Merge pull request #82 from splunk/migrate_app_es_soc

Moving our package to this repo from app_es_soc

.circleci/config.yml

@@ -3,6 +3,8 @@
 # Check https://circleci.com/docs/2.0/language-python/ for more details
 #
 
+version: 2.1
+
 dependencies:
   cache_directories:
     - "~/.apt-cache"
@@ -15,8 +17,6 @@ apt-run: &apt-install
     sudo apt update -qq
     sudo apt install -y enchant python-dev -qq
 
-version: 2.1
-
 executors:
   content-executor:
     docker:
@@ -36,7 +36,7 @@ jobs:
                 git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
             fi
       - restore_cache:
-          key: deps1-{{ .Branch }}-{{ checksum "security-content/requirements.txt" }}
+          key: virtualenv
       - run: *apt-install
       - run:
           name: install python dependencies
@@ -46,29 +46,17 @@ jobs:
             virtualenv --python=/usr/bin/python2.7 --clear venv
             source venv/bin/activate
             pip install -q -r requirements.txt
+      - save_cache:
+          key: virtualenv
+          paths:
+            - "/security-content/venv"
       - run:
           name: run validate
           command: |
             cd security-content
             source venv/bin/activate
-
             python bin/validate.py --path . --verbose
-      - run:
-          name: run generate
-          command: |
-            cd security-content
-            source venv/bin/activate
-            python bin/generate.py --path . --output src --storiesv1 --use_case_lib -v
-      - persist_to_workspace:
-          root: security-content/src/default
-          paths:
-              - use_case_library.conf
-              - analytic_stories.conf
-              - savedsearches.conf
-      - save_cache:
-          key: deps1-{{ .Branch }}-{{ checksum "security-content/requirements.txt" }}
-          paths:
-            - "venv"
+
   build-sources:
     executor: content-executor
     steps:
@@ -80,27 +68,141 @@ jobs:
             else
                 git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
             fi
-      - attach_workspace:
-        # Must be absolute path or relative path from working_directory
-          at: ~/repo/updated
+      - run: *apt-install
       - run:
-          name: store updated analyticstories.conf
+          name: install python dependencies
           command: |
             cd security-content
-            cp -v ~/repo/updated/* src/default/
-            git config credential.helper 'cache --timeout=120'
-            git config user.email "research@splunk.com"
-            git config user.name "research bot"
-            git config --global push.default simple
-            git add src/default/*
-            git commit --allow-empty -m "updating src files [ci skip]"
-            # Push quietly to prevent showing the token in log
-            git push https://${GITHUB_TOKEN}@github.com/splunk/security-content.git ${CIRCLE_BRANCH}
-            tar -czvf content-pack.tar.gz src/*
+            rm -rf venv
+            virtualenv --python=/usr/bin/python2.7 --clear venv
+            source venv/bin/activate
+            pip install -q -r requirements.txt
+      - run:
+          name: run generate
+          command: |
+            cd security-content
+            source venv/bin/activate
+            python bin/generate.py --path . --output package --storiesv1 --use_case_lib -v
+      - run:
+          name: update version and build number
+          command: |
+            cd security-content
+            # check if tag is set, get build number from the tag if set
+            if [ -z "${CIRCLE_TAG}" ]; then
+                CONTENT_VERSION=$(grep -oP "(\d.\d.\d+$)" package/default/content-version.conf)
+                echo "detected content version: $CONTENT_VERSION"
+            else
+                CONTENT_VERSION=$(echo $CIRCLE_TAG | grep -oP "\d.\d.\d+")
+                echo "content version: $CONTENT_VERSION, set by tag: $CIRCLE_TAG"
+            fi
+            # update build number and version
+            sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" package/default/app.conf
+            sed -i "s/version = .*$/version = $CONTENT_VERSION/g" package/default/app.conf
+            sed -i "s/version = .*$/version = $CONTENT_VERSION/g" package/default/content-version.conf
+            tar -czf content-pack-build.tar.gz package/*
       - persist_to_workspace:
           root: security-content/
           paths:
-              - content-pack.tar.gz
+              - content-pack-build.tar.gz
+  build-package:
+    executor: content-executor
+    steps:
+      - attach_workspace:
+          at: ~/dist
+      - run:
+          name: grab splunk packaging toolkit
+          command: |
+            curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-1.0.0.tar.gz -o ~/splunk-packaging-toolkit-latest.tar.gz
+            mkdir ~/slim-latest
+            tar -zxf ~/splunk-packaging-toolkit-latest.tar.gz -C ~/slim-latest --strip-components=1
+      - run:
+          name: install splunk packaging toolkit (slim)
+          command: |
+              cd ~/slim-latest
+              sudo pip install --upgrade pip setuptools
+              sudo pip install virtualenv
+              virtualenv --python=/usr/bin/python2.7 --clear venv
+              source venv/bin/activate
+              pip install .
+      - run:
+          name: create a .spl for this build using slim
+          command: |
+              source ~/slim-latest/venv/bin/activate
+              cd ~/dist
+              tar -zxf content-pack-build.tar.gz
+              CONTENT_VERSION=$(grep -oP "(\d.\d.\d+$)" package/default/content-version.conf)
+              mv package DA-ESS-ContentUpdate
+              mkdir upload
+              slim package -o upload DA-ESS-ContentUpdate
+              cp upload/*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz
+      - store_artifacts:
+          path: ~/dist/upload
+          destination: package/
+      - persist_to_workspace:
+          root: ~/dist
+          paths:
+             - DA-ESS-ContentUpdate-latest.tar.gz
+  run-appinspect:
+    executor: content-executor
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run: *apt-install
+      - run:
+          name: grab appinspect
+          command: |
+            curl -Ls http://dev.splunk.com/goto/appinspectdownload -o appinspect-lastest.tar.gz
+            mkdir appinspect-latest
+            tar -zxf appinspect-lastest.tar.gz -C appinspect-latest --strip-components=1
+      - run:
+          name: install app inspect
+          command: |
+            cd appinspect-latest
+            rm -rf venv
+            sudo pip install --upgrade pip setuptools
+            sudo pip install virtualenv
+            virtualenv --python=/usr/bin/python2.7 --clear venv
+            source venv/bin/activate
+            pip install .
+      - run:
+          name: run app inspect
+          command: |
+            cd appinspect-latest
+            source venv/bin/activate
+            splunk-appinspect inspect ~/DA-ESS-ContentUpdate-latest.tar.gz --included-tags=cloud --max-messages=all
+      - persist_to_workspace:
+          root: ~/
+          paths:
+             - DA-ESS-ContentUpdate-latest.tar.gz
+  update-sources-github:
+    executor: content-executor
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run: *apt-install
+      - run:
+          name: checkout repo
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "" ]; then
+                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
+            else
+                git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
+            fi
+            # configure git to prep for commit
+            git config credential.helper 'cache --timeout=120'
+            git config user.email "research@splunk.com"
+            git config user.name "research bot"
+            it config --global push.default simple
+            git add package/default/*
+            git commit --allow-empty -m "updating package files [ci skip]"
+            # Push quietly to prevent showing the token in log
+            git push https://${GITHUB_TOKEN}@github.com/splunk/security-content.git ${CIRCLE_BRANCH}
+            CONTENT_VERSION=$(echo $CIRCLE_TAG | grep -oP "\d.\d.\d+")
+            tar -czf DA-ESS-ContentUpdate-${CONTENT_VERSION}.tar.gz package/*
+      - persist_to_workspace:
+          root: security-content/
+          paths:
+              - DA-ESS-ContentUpdate-${CONTENT_VERSION}.tar.gz
   publish-github-release:
     docker:
       - image: cibuilds/github:0.10
@@ -110,18 +212,48 @@ jobs:
       - run:
           name: publish release on github
           command: |
-            ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ~/repo/updated/content-pack.tar.gz
+            CONTENT_VERSION=$(echo $CIRCLE_TAG | grep -oP "\d.\d.\d+")
+            ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ~/repo/updated/DA-ESS-ContentUpdate-${CONTENT_VERSION}.tar.gz
 workflows:
   version: 2.1
   validate-and-build:
     jobs:
       - validate-content:
+        # build always
           filters:
             tags:
               only: /.*/
       - build-sources:
           requires:
             - validate-content
+        # build always
+          filters:
+            tags:
+              only: /.*/
+      - build-package:
+          requires:
+            - validate-content
+            - build-sources
+        # build always
+          filters:
+            tags:
+              only: /.*/
+      - run-appinspect:
+          requires:
+            - validate-content
+            - build-sources
+            - build-package
+        # build always
+          filters:
+            tags:
+              only: /.*/
+      - update-sources-github:
+          requires:
+            - validate-content
+            - build-sources
+            - build-package
+            - run-appinspect
+        # only update sources in develop
           filters:
             tags:
               only: /^v.*/
@@ -131,6 +263,10 @@ workflows:
           requires:
             - validate-content
             - build-sources
+            - build-package
+            - run-appinspect
+            - update-sources-github
+        # only release when there is a tag
           filters:
             tags:
               only: /^v.*/

.pre-commit-config.yaml

@@ -4,6 +4,7 @@ repos:
   hooks:
     - id: trailing-whitespace
     - id: check-executables-have-shebangs
+      exclude: 'package/bin/da_ess_contentupdate/|package/bin/splunklib/|venv/'
     - id: check-json
     - id: check-symlinks
     - id: check-yaml
@@ -11,4 +12,5 @@ repos:
       args: [--autofix]
     - id: flake8
       args: [--max-line-length=131]
+      exclude: 'package/bin/da_ess_contentupdate/|package/bin/splunklib/|venv/|package/bin/escu_contextualize.py|package/bin/escu_investigate.py|package/bin/runstory.py'
     - id: requirements-txt-fixer

README.md

@@ -23,7 +23,7 @@ Can be consumed using:
 * [investigations/](investigations/) - splunk, and phantom investigation content that are used in stories
 * [responses/](responses/) - automated splunk and phantom responses that are used in stories
 * [baselines/](baselines/) - phantom and Splunk baseline needed to support detections in stories
-* [src/](src/) - splunk content app source files, includes lookups, binaries, and defaul config files
+* [package/](package/) - splunk content app source files, includes lookups, binaries, and defaul config files
 * [bin/](bin/) - where all binaries to produce, and test content lives
 * [spec/](spec/) - location of all spec files that describe ESCU content
 * [docs/](docs/) - documentation for all of the spec files

bin/generate.py

@@ -253,6 +253,10 @@ def generate_investigations(REPO_PATH, detections, stories):
                     server = phantom['phantom_server']
                     playbook = phantom['playbook_name']
                     playbook_url = phantom['playbook_url']
+                    earliest_time = phantom['schedule']['earliest_time']
+                    latest_time = phantom['schedule']['latest_time']
+                    cron = phantom['schedule']['cron_schedule']
+                    search = 'CONSTRUCT DETECTION SEARCH HERE'
                 except KeyError as e:
                     sys.exit("ERROR: \"{1}\" missing key {0} with error:\n{1}".format(e, name, e))
 

package/README.md

@@ -0,0 +1,31 @@
+# Fidelity
+
+The fidelity of a narrative describes the ratio of signal (valid/positive) to noise (invalid/false positive) anticipated based on field experience.
+
+* High - This indicates a relatively high signal to noise ratio, and therefore a lower likelihood of false positives, and it should not require additional searches to validate it.
+
+Example:
+
+```
+
+sourcetype=WinEventLog:* EventCode=4728
+```
+
+
+
+* Low - This indicates a relatively low signal to noise ratio, and therefore a higher likelihood of false positives.  Confidence in the output can be increased through other means (i.e. cross-correlation and/or subsequent searches).
+
+Example:
+
+```
+
+url=* | eval url_length = len(url) | where url_length > 256
+```
+* Moderate - This indicates an unpredictable signal to noise ratio with a bias towards signal, and therefore a higher likelihood of false positives than high. Confidence in the output can be increased through other means (i.e. cross-correlation and/or subsequent searches).
+
+Example:
+
+```
+http_user_agent = "*nullptr*"
+```
+

package/README/alert_actions.conf.spec

@@ -0,0 +1,85 @@
+# Copyright (C) 2009-2016 Splunk Inc. All Rights Reserved.
+#
+# This file contains additional options for an alert_actions.conf file.
+#
+# To learn more about configuration files (including precedence) please see the documentation
+# located at http://www.splunk.com/base/Documentation/latest/Admin/Aboutconfigurationfiles
+#
+[escu]
+
+enabled = [true|false|0|1]
+    * Whether or not this use-case is enabled.
+    * This exists so that we are a true noop for scheduled searches.
+    * action.usecase=0 action.usecase.enabled=1
+    * Required.
+    * Defaults to false.
+
+version = [string]
+    * Version of this search
+
+asset_at_risk = [string]
+    * The type of asset that is at risk from the behavior this search is attempting to find
+    * Defaults to None
+
+category = [string]
+    * A description of the category that this use-case falls into
+    * Defaults to None
+
+channel = [string]
+    * The name of the channel the search belongs to
+
+confidence = [low|medium|high]
+    * A description of the confidence value
+    * Valid values are: low, medium, high
+    * Defaults to None
+
+creation_time = [datetime]
+    * The date & time that the search was first created
+    * The date-time should be formatted an epoch time (in GMT)
+
+datamodels = [json]
+    * A JSON list of the data models used by this search
+    * Defaults to None
+
+eli5 = [string]
+    * Text explaining this search to a 5 year old
+    * Defaults to None
+
+full_search_name = [string]
+    * The entire search name
+    * Defaults to None
+
+how_to_implement = [string]
+    * Text discussing what needs to be done to implement this search and any local modifications that can be performed
+    * Defaults to None
+
+known_false_positives = [string]
+    * A description of cases in which this use-case may generate false positive alerts
+    * Defaults to None
+
+mappings = [json]
+    * A JSON list of the kill chain phases this search covers
+    * Defaults to None
+
+modification_time = [datetime]
+    * The date that the search was last modified
+    * The date-time should be formatted an epoch time (in GMT)
+
+remediation = [string]
+    * A high-level description of how the issue described by this use-case can be remediated.
+    * Defaults to None
+
+providing_technologies = [json]
+    * A JSON list of the technology examples that can be used to gather data to power this search
+    * Defaults to None
+
+analytic_story = [json]
+    * A JSON list of the use cases this search applies to
+    * Defaults to None
+
+earliest_time_offset = [integer]
+    * Time in seconds before event time that the search should cover
+
+latest_time_offset = [integer]
+    * Time in seconds after event time that the search should cover
+

package/README/analytic_story.conf.spec

@@ -0,0 +1,100 @@
+# Copyright (C) 2009-2016 Splunk Inc. All Rights Reserved.
+#
+# This file contains all possible options for a usecases.conf file.  Use this file to define a use-case.
+#
+# To learn more about configuration files (including precedence) please see the documentation
+# located at http://www.splunk.com/base/Documentation/latest/Admin/Aboutconfigurationfiles
+#
+[<analytic_story_name>]
+category = [string]
+    * The category of the analytic story
+    * Defaults to None
+
+creation_time = [datetime]
+    * The date & time that the analytic story was first created
+    * The date-time should be formatted an epoch time (in GMT)
+
+data_models = [json]
+    * A JSON list of the data models used by the analytic story
+    * Defaults to None
+
+description = [string]
+    * A bried description of the analytic story
+    * Defaults to None
+
+id = [string]
+    * A description of the analytic story
+    * Defaults to None
+
+mappings = [json]
+    * A JSON dictionary of the different mappings this story maps to
+    * See appendix B for the format of this field
+    * Defaults to None
+
+modification_time = [datetime]
+    * The date & time that the analytic story was last modified
+    * The date-time should be formatted an epoch time (in GMT)
+
+narrative = [string]
+    * A longer narrative of the analytic story that describes the detection searches, any support searches,
+    * and the corresponding contextual and investigative searches
+    * Defaults to None
+
+references = [json]
+    * A JSON list of references for this story
+    * Defaults to None
+
+detection_searches = [json]
+    * A JSON list of the detection searches that the analytic story applies to.
+    * See appendix A for the format of this field
+    * Defaults to None
+
+investigative_searches = [json]
+    * A JSON list of the investigative searches that the analytic story applies to.
+    * See appendix A for the format of this field
+    * Defaults to None
+
+contextual_searches = [json]
+    * A JSON list of contextual searches that the analytic story applies to.
+    * See appendix A for the format of this field
+    * Defaults to None
+
+support_searches = [json]
+    * JSON list of support searches that the analytic story applies to.
+    * See appendix A for the format of this field
+    * Defaults to None
+
+providing_technologies = [json]
+    * A JSON list of example technologies that can be used to capture the data needed for the analytic story
+    * Defaults to None
+
+version = [int]
+    * An integer indicating which revision of the analytic story this is
+    * This value should start with one and increase for each release
+
+###### Appendix A: *_searches Specification #######
+# This can just be a list of saved search names. However, this also supports a hierarchical structure to denote searches that rely on other searches.
+#
+# A non-hierarchical version would look like this:
+#      [ "search1", "search2" ]
+#
+# A hierarchical version would look like this:
+#[
+#       "search1": [ "search1a", "search1b" ],
+#       "search2": [ "search2a" ]
+#]
+
+
+###### Appendix B: Mappings Specification #######
+#
+# This is a dictionary of the different mappings this analytic story maps to.
+# The mapping will be the key, and the value will be an array of the labels it applies to
+#
+# Example:
+# {
+#   "kill_chain_phase": ["Delivery", "Command and Control"],
+#   "sans cis": ["CIS 9", "CIS 12"],
+#   "att&ck": ["Command and Control"]
+# }
+#
+

package/README/essoc_story_detail.txt

@@ -0,0 +1,15 @@
+The Analytic Story Details dashboard renders all the details of the content related to a specific analytic story which
+can be chose via the drop down
+
+Each analytic story has attributes associated with it and the following:
+______________________________________________________________________
+
+
+	Analytic Story: name of the analytic story
+	Description	; description of the analytic story
+	Search Name	: The name of the searches belonging to the chosen analytic story
+	Search	: The search query which looks for an attack pattern corresponding to the analytic story
+	Search Description: The description of the search query
+	Asset Type: The analytic story specifies what asset in the infrastructure may be compromised
+	Category: The category that the search belongs to (malware, vulnerabilities, best practices, abuse)
+	Kill Chain Phase: The kill chain phase of the attack that the search is after.

package/README/essoc_summary.txt

@@ -0,0 +1,24 @@
+The ES_SOC Summary Dashboard provides you a summarized view of the analytic story contents of the ES-SOC app.
+The dashboard has the following panels gives you following details
+
+1) Analytic story Summary
+ 	- Total Analytic Stories : The total number of Analytic stories in the ES-SOC application
+ 	- Total Searches: The total number of searches in ES-SOC
+ 	- Searches added last week: Number of searches added to ES-SOC in the last week.
+
+ 2) Analytic story Category: This dashboard panel summarizes the categories of the searches that the ES-SOC app contains. The categories of the analytic stories are as follow
+  	-Malware: These searches detect specific malware behavior for a particular phase of the attack kill chain. E.g. a malware’s delivery method via email or a malware’s installation behavior via registry key changes
+  	-Vulnerability: These searches detect behavior or a signature of a vulnerable software in use. These searches are not designed to replace vulnerability management or scanning systems. The purpose of these searches is to discover a vulnerability through side effects or behaviors.
+  	-Abuse: Some actions can be deemed malicious because they are unexpected, violate corporate policy or are significantly different than the actions of other users. E.g. A USB disk that is seen on multiple systems or a user that uploads excessive files to a cloud service or a database query that dumps an entire table
+  	-Best Practices:  Searches that correspond to specific guidelines from organizations like SANS or OWASP
+
+ 3) Kill Chain phases: Every analytic story has one or more searches which look for a certain kind of attack pattern/behavior. These searches have an attribute which essentially tells you what Kill chain phase does the search correspond to.
+    The numbers on the dashboard represents the number of searches correponding to each kill chain phase
+
+ 4) Analytic story table: This table gives the user a comprehensive view of some of the details of the analytic story. Some of the listed attributes are:
+  	- Analytic Story : The name of the analytic story
+  	- Description: The description of the analyttic story
+  	- Search names: The name of the searches in each analytic story
+  	- Datamodels: The name of the datamodel that the search is querying against.
+  	- Technology Examples: This field represent some examples related to the technologies required to populate the datamodels(Nessues, Cisco Firewall,etc)
+  	- Kill chain phase: The name of the kill chain phase that the search belongs to

package/README/essoc_usage_dashboard.txt

@@ -0,0 +1,51 @@
+######################
+ESSOC Usage Dashboard#
+######################
+
+The ESSOC Usage dashboard is designed to provide high-level insight into the usage of the ES-SOC app. It is suitable for display when providing feedback to the Splunk team or for identifying how the ES-SOC app is being used. This dashboard has two time selectors that work independently - the top time selector determines the search time range for all the single-value. And the lower time selector, determines the time range for the usage table.
+
+IMPORTANT: The user loading this dashboard must have permission to search the _audit index
+
+##################
+#Dashboard panels#
+##################
+
+Searches Ran
+
+The total number of searches in ES-SOC that were executed. This number includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch <ESSOC search_name> ‘ syntax
+
+Unique Searches
+
+The unique/distinct searches executed on the deployment. This is equivalent to the distinct count of searches run in the ES-SOC app.
+
+Most Run
+
+The total number of searches in ES-SOC that were executed. This number includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch <ESSOC search_name> ‘ syntax.
+
+Ad hoc Searches
+
+The total number of searches run from the search bar using the '| savedsearch <ESSOC search_name> ‘ syntax.
+
+Scheduled
+
+The total number of ESSOC searches run that were scheduled.
+
+Most Active User
+
+The user who executed the highest number/count of searches. This calculation includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch <ESSOC search_name> ‘ syntax.
+
+Search Run Time (seconds)
+
+Total run time of all searches executed in seconds. This calculation includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch <ESSOC search_name> ‘ syntax.
+
+Average Run Time (seconds)
+
+Average run time of all searches executed in seconds. This calculation includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch <ESSOC search_name> ‘ syntax.
+
+Max Run Time (seconds)
+
+The run time of the longest running search. This calculation includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch <ESSOC search_name> ‘ syntax.
+
+Search summary
+
+This table provides details on each search that was executed in the ESSOC app.

package/README/savedsearches.conf.example

@@ -0,0 +1,38 @@
+[SMTP PII file sent]
+
+# Alert action stuff
+action.usecase                       = 0
+action.usecase.enabled               = 1
+action.usecase.category              = pii_monitoring
+action.usecase.reference             = https://en.wikipedia.org/wiki/Extended_SMTP
+action.usecase.kill_chain_phases     = Weaponization,Delivery
+action.usecase.search_score          = 5
+action.usecase.confidence            = high
+action.usecase.known_false_positives = Some Unix SMTP servers will use enhanced SMTP commands that can trigger this event even though no malicious activity is present
+action.usecase.remediation           = Block this activity at your email server using magic
+
+# Normal saved search stuff
+cron_schedule                        = */30 * * * *
+disabled                             = False
+dispatch.earliest_time               = -24h
+dispatch.latest_time                 = +0s
+enableSched                          = True
+is_visible                           = false
+search                               = | `tstats` values(sourcetype) from datamodel=Change_Analysis.All_Changes by All_Changes.Endpoint_Changes.Filesystem_Changes.file_hash | `tstats` append=true values(sourcetype) from datamodel=Email.All_Email by All_Email.file_hash | `tstats` append=true values(sourcetype) from datamodel=Malware.Malware_Attacks by Malware_Attacks.file_hash | `tstats` append=true values(sourcetype) from datamodel=Updates.Updates by Updates.file_hash | rename All_Changes.Endpoint_Changes.Filesystem_Changes.* as *,All_Email.* as *,Malware_Attacks.* as *,Updates.* as * | stats values(sourcetype) as orig_sourcetypes by file_hash | lookup threatintel_by_file_hash file_hash OUTPUT | search threat_collection_key=* | `zipexpand_threat_matches` | `makesv(orig_sourcetypes)`
+description							 = This search does PII monitoring
+
+
+## EXAMPLE EMPTY FIELDS
+action.usecase                       = 0
+action.usecase.enabled               = 1
+action.usecase.category              =
+action.usecase.reference             =
+action.usecase.kill_chain_phases     =
+action.usecase.search_score          =
+action.usecase.confidence            =
+action.usecase.known_false_positives =
+action.usecase.remediation           =
+search                               =
+description                          =
+dispatch.earliest_time               = -24h@h
+dispatch.latest_time                 = 0

package/appserver/static/analytic_story_details.css

@@ -0,0 +1,186 @@
+/* .rTable {
+  	display: table;
+  	width: 100%;
+}
+
+.rTableRow {
+  	display: table-row;
+}
+
+.rTableHeading {
+  	display: table-header-group;
+  	background-color: #ddd;
+}
+
+.rTableCell, .rTableHead {
+  	display: table-cell;
+  	padding: 3px 10px;
+  	//border: 1px solid #999999;
+}
+
+.rTableLeftCell {
+  	display: table-cell;
+  	padding: 3px 10px;
+  	//border: 1px solid #999999;
+	width: 200px;
+}
+
+.rTableHeading {
+  	display: table-header-group;
+  	background-color: #ddd;
+  	font-weight: bold;
+}
+
+.rTableFoot {
+  	display: table-footer-group;
+  	font-weight: bold;
+  	background-color: #ddd;
+}
+
+.rTableBody {
+  	display: table-row-group;
+} */
+
+h1 {
+	font-size: 24px;
+    font-weight: 200;
+    margin: 0;
+}
+
+h3 {
+	padding-left: 10px;
+}
+
+.as_title_attr_bar {
+	padding-left: 15%;
+    background-color: #eee;
+    height: 40px;
+	line-height: 40px;
+	margin-bottom: 7px;
+}
+
+.as_title_attr {
+	float: left;
+	margin-right: 5%;
+	padding-right: 20px;
+	font-size: 14px;
+}
+
+.as_search_accordion {
+	width: 100%;
+	margin-top: 10px;
+}
+
+.as_story_details {
+	display: flex;
+	max-height: 500px;
+	clear: both;
+}
+
+.as_story_details_left_col {
+	float: right;
+	width: 34%;
+	display: block;
+	overflow: scroll;
+	padding: 10px;
+	margin: 10px;
+	border: 1px solid #ddd;
+}
+
+.as_story_details_right_col {
+	float: left;
+	width: 60%;
+	overflow: scroll;
+	padding: 10px;
+	margin: 10px;
+	border: 1px solid #ddd;
+}
+
+.as_left_attr{
+	display: inline-block;
+	width: 100%;
+}
+
+.as_story_detail_left_attr_label {
+	float: left;
+	width: 30%;
+	padding-top: 2%;
+}
+
+.as_story_detail_left_attr {
+	margin-top: 5px;
+	float: left;
+	width: 70%;
+}
+
+.value_label {
+	float: left;
+    margin-right: 5px;
+    background-color: #eee;
+    padding: 4px;
+	border-radius: 6px;
+    margin-bottom: 5px;
+}
+
+.as_story_detail_right_attr_label {
+	margin-bottom: 7px;
+}
+
+.search_content {
+	display: flex;
+	clear: both;
+}
+
+.search_left_panel {
+	float: left;
+	width: 70%;
+	margin: 10px;
+	padding: 10px;
+	border: 1px solid #ddd;
+}
+
+.search_right_panel {
+	float: left;
+	width: 25%;
+	margin: 10px;
+	padding: 10px;
+	border: 1px solid #ddd;
+}
+
+.search_left_attr {
+	margin: 10px;
+}
+
+.search_right_attr {
+	margin: 10px;
+	display: table;
+}
+
+.search_string{
+	padding: 10px;
+    background-color: #ddd;
+    border: 1px solid #aaa;
+    border-radius: 2px;
+}
+
+.data_model_tag {
+	background-color: #11a88b;
+}
+
+.kill_chain_tag {
+	background-color: #ed8440;
+}
+
+.attack_tag {
+	background-color: #3863a0;
+	color: #eee;
+}
+
+.heading-story {
+	width: 80%;
+	float: left;
+}
+
+.run_story_btn {
+	float: right;
+}

package/appserver/static/analytic_story_details.js

@@ -0,0 +1,601 @@
+require([
+    'underscore',
+    'jquery',
+    'splunkjs/mvc',
+    'splunkjs/mvc/searchmanager',
+    'splunkjs/mvc/searchbarview',
+    'splunkjs/mvc/tableview',
+    'splunk.util',
+    '../app/DA-ESS-ContentUpdate/js/lib/showdown.min',
+    '../app/DA-ESS-ContentUpdate/js/lib/jquery-ui/jquery-ui',
+    'css!../app/DA-ESS-ContentUpdate/js/lib/jquery-ui/jquery-ui.css',
+    'css!../app/DA-ESS-ContentUpdate/analytic_story_details.css',
+    'splunkjs/mvc/simplexml/ready!'
+], function(_, $, mvc, SearchManager, SearchBarView, TableView, splunkUtil, showdown) {
+
+    let tokenModel = mvc.Components.get("default");
+    let renderedComponents = [];
+
+    let templ = `
+                 <div class="as_title_attr_bar">
+                    <div class="as_title_attr">
+                        <strong>Category: </strong><span id="as_label_category"></span>
+                    </div>
+                    <div class="as_title_attr">
+                        <strong>Version: </strong><span id="as_label_version"></span>
+                    </div>
+                    <div class="as_title_attr">
+                        <strong>Created: </strong><span id="as_label_created"></span>
+                    </div>
+                    <div class="as_title_attr">
+                        <strong>Modified: </strong><span id="as_label_modified"></span>
+                    </div>
+                </div>
+                <div class="headline_story">
+                    <div class="heading-story">
+                        <h1 id="story_heading"></h1>
+                    </div>
+                    <div class="run_story_btn">
+                        <button class="btn btn-primary run-story">Run Analytics</button>
+                    </div>
+                </div>
+                <div class="as_story_details">
+                    <div class="as_story_details_right_col">
+                        <div class="as_story_detail_right_attr_label">
+                            <strong>Description: </strong>
+                        </div>
+                        <div class="as_story_detail_right_attr_label">
+                            <span id="description"></span>
+                        </div>
+                        <div class="as_story_detail_right_attr_label">
+                            <strong>Narrative: </strong>
+                        </div>
+                        <div class="as_story_detail_right_attr_label narrative_value">
+                            <span id="narrative"></span>
+                        </div>
+                    </div>
+                    <div class="as_story_details_left_col">
+                        <div class="as_left_attr">
+                            <div class="as_story_detail_left_attr_label">
+                                <strong>ATT&CK: </strong>
+                            </div>
+                            <div class="as_story_detail_left_attr" id="mitre_attack">
+                            </div>
+                        </div>
+                        <div class="as_left_attr">
+                            <div class="as_story_detail_left_attr_label">
+                                <strong>Kill Chain Phases: </strong>
+                            </div>
+                            <div class="as_story_detail_left_attr kill_chain_phases" id="kill_chain_phases">
+                            </div>
+                        </div>
+                        <div class="as_left_attr">
+                            <div class="as_story_detail_left_attr_label">
+                                <strong>CIS Controls: </strong>
+                            </div>
+                            <div class="as_story_detail_left_attr" id="cis_20">
+                            </div>
+                        </div>
+                        <div class="as_left_attr">
+                            <div class="as_story_detail_left_attr_label">
+                                <strong>Data Model: </strong>
+                            </div>
+                            <div class="as_story_detail_left_attr" id="data_model">
+                            </div>
+                        </div>
+                        <div class="as_left_attr">
+                            <div class="as_story_detail_left_attr_label">
+                                <strong>Technologies: </strong>
+                            </div>
+                            <div class="as_story_detail_left_attr" id="technology">
+                            </div>
+                        </div>
+                        <div class="as_left_attr">
+                            <div class="as_story_detail_left_attr_label">
+                                <strong>References: </strong>
+                            </div>
+                            <div class="as_story_detail_left_attr" id="references">
+                            </div>
+                        </div>
+                    </div>
+                 </div>
+                 <div class="as_search_details">
+                    <h2>
+                        Analytic Story Searches
+                    </h2>
+                    <div id="accordion">
+                        <h3>Detection</h3>
+                        <div>
+                            <div id="search_detection">
+                            </div>
+                        </div>
+                        <h3>Context</h3>
+                        <div>
+                            <div id="search_contextual">
+                            </div>
+                        </div>
+                        <h3>Investigative</h3>
+                        <div>
+                            <div id="search_investigative">
+                            </div>
+                        </div>
+                        <h3>Support</h3>
+                        <div>
+                            <div id="search_support">
+                            </div>
+                        </div>
+                    </div>
+                 </div>
+                `;
+
+    $('#analytic_story_details').html(_.template(templ));
+
+    if (tokenModel.get('analytic_story_name')) {
+        fetchAnalyticStoryDetails(tokenModel.get('analytic_story_name'));
+    }
+
+    tokenModel.on("change:analytic_story_name", function(model, value, options) {
+        fetchAnalyticStoryDetails(value);
+    });
+
+    function fetchAnalyticStoryDetails(asName) {
+        let epoch = (new Date).getTime();
+        let searchGetAnalyticStoryData = new SearchManager({
+            id: epoch,
+            earliest_time: "-1h@h",
+            latest_time: "now",
+            cache: false,
+            search: "| rest /services/configs/conf-analytic_stories splunk_server=local count=0 | search title=\"" + asName + "\" | spath input=providing_technologies path={} output=tex | spath input=reference path={} output=ref | spath input=data_models path={} output=dm | table title, category, description, version, mappings, creation_date, modification_date, dm, narrative, tex, ref"
+        });
+
+        $('.run-story').unbind('click');
+
+        $('.run-story').on('click', function(evt) {
+            splunkUtil.redirect_to('/app/DA-ESS-ContentUpdate/search', {
+                q: `| runstory story="${asName}" | table name, num_search_results, description, kill_chain_phases, mitre_attack`,
+                earliest: "-60m",
+                latest: "now",
+
+            }, window.open(), true);
+        });
+
+        let asSearch = splunkjs.mvc.Components.getInstance(epoch);
+        let asResults = asSearch.data("results", {
+            count: 0
+        });
+        asResults.on("data", function() {
+            let as_attributes = {};
+            let fields = asResults.data().fields;
+            let rows = asResults.data().rows;
+
+            for (let i = 0; i < fields.length; i++) {
+                as_attributes[fields[i]] = rows[0][i];
+            }
+            renderStoryAttributes(as_attributes);
+        });
+
+        var searchGetSearchesData = new SearchManager({
+            id: "s" + epoch,
+            earliest_time: "-1h@h",
+            latest_time: "now",
+            cache: false,
+            search: "| rest /services/saved/searches splunk_server=local count=0 | spath input=action.escu.analytic_story path={} output=uc | search uc = \"" + asName + "\" | spath input=action.escu.data_models path={} output=dm | spath input=action.escu.providing_technologies path={} output=tex | table action.escu.full_search_name, search, description, action.escu.search_type, action.escu.how_to_implement, action.escu.eli5, action.escu.version, action.escu.mappings, dm, tex, action.escu.asset_at_risk, action.escu.confidence, action.escu.known_false_positives, updated, action.escu.modification_date, action.escu.creation_date "
+
+        });
+        var searchesSearch = splunkjs.mvc.Components.getInstance("s" + epoch);
+        var searchesResults = searchesSearch.data("results", {
+            count: 0
+        });
+
+
+        searchesResults.on("data", function() {
+            let asSearchAttr = [];
+            var fields = searchesResults.data().fields;
+            var rows = searchesResults.data().rows;
+
+            for (let i = 0; i < rows.length; i++) {
+                let searchObj = {};
+                for (let j = 0; j < fields.length; j++) {
+                    searchObj[fields[j]] = rows[i][j];
+                }
+                asSearchAttr.push(searchObj);
+            }
+            renderSearches(asSearchAttr);
+        });
+    }
+
+    function renderStoryAttributes(asAttributes) {
+        let converter = new showdown.Converter();
+        let mappings = JSON.parse(asAttributes.mappings);
+        $('#as_label_category').html(asAttributes.category);
+        $('#as_label_version').html(asAttributes.version);
+        $('#as_label_created').html(asAttributes.creation_date);
+        $('#as_label_modified').html(asAttributes.modification_date);
+        $('#story_heading').html(asAttributes.title);
+        $('#attack').html(mappings.mitre_attack);
+        $('#narrative').html(converter.makeHtml(asAttributes.narrative));
+        $('#description').html(converter.makeHtml(asAttributes.description));
+        $('#mitre_attack').html(getValueLabels(mappings.mitre_attack, 'attack_tag'));
+        $('#data_model').html(getValueLabels(asAttributes.dm, 'data_model_tag'));
+        $('#technology').html(getValueLabels(asAttributes.tex));
+        $('#kill_chain_phases').html(getValueLabels(mappings.kill_chain_phases, 'kill_chain_tag'));
+        $('#cis_20').html(getValueLabels(mappings.cis20));
+        $('#references').html(getReferenceURLS(asAttributes.ref));
+    }
+
+    function getReferenceURLS(refs) {
+        if (refs === null) {
+            return " ";
+        } else {
+            let refsResult = ``;
+            if (Array.isArray(refs)) {
+                refs.map(ref => {
+                    refsResult = refsResult + `<a href="${ ref }">${ ref }</a><br />`;
+                });
+            } else {
+                refsResult = refsResult + `<a href="${ refs }">${ refs }</a><br />`
+            }
+
+            return refsResult;
+        }
+    }
+
+    function renderSearches(asSearches) {
+        clearSearchView();
+        let i = 0;
+        let converter = new showdown.Converter();
+        asSearches.forEach(search => {
+            i++;
+            let epoch = (new Date).getTime();
+            let searchID = `#search${ i }`;
+            let resultID = `#result${ i }`;
+            let searchSelector = `search${ i }`;
+            let controlID = `as_search${ i }`
+            let resultsControlID = `as_results_search${ i }`;
+            let btnID = `btn_es_${i}`;
+
+            let searchPanel = `
+                            <h3>${ search['action.escu.full_search_name'] }</h3>
+                            <div class="search_content" id="${searchSelector}-content">
+                                <div class="search_left_panel">
+                                    <button class="configure_in_es btn btn-primary" id="${ btnID }" data-search-type="${search['action.escu.search_type']}"  data-search-name="${ search['action.escu.full_search_name'] }">Configure</button>
+                                    <div class="search_left_attr">
+                                        <div class="search_left_attr_label">
+                                            <strong>Description</strong>
+                                        </div>
+                                        <div class="search_left_attr_value">
+                                            ${ converter.makeHtml(search['description']) }
+                                        </div>
+                                    </div>
+                                    <div id="${searchSelector}-eli5">
+                                    </div>
+                                    <div class="search_left_attr">
+                                        <div class="search_left_attr_label">
+                                            <strong>Search</strong>
+                                        </div>
+                                        <div class="search_left_attr_value ${ controlID }">
+                                        </div>
+                                        <div class="search_left_attr_value ${ resultsControlID }">
+                                        </div>
+                                    </div>
+                                    <div class="search_left_attr">
+                                        <div class="search_left_attr_label">
+                                            <strong>How to Implement</strong>
+                                        </div>
+                                        <div class="search_left_attr_value">
+                                            ${ converter.makeHtml(search['action.escu.how_to_implement']) }
+                                        </div>
+                                    </div>
+                                    <div class="search_left_attr">
+                                        <div class="search_left_attr_label">
+                                            <strong>Known False Positives</strong>
+                                        </div>
+                                        <div class="search_left_attr_value">
+                                            ${ converter.makeHtml(search['action.escu.known_false_positives']) }
+                                        </div>
+                                    </div>
+                                </div>
+                                <div class="search_right_panel">
+                                    <div class="search_right_attr data_model_srch_attr">
+                                        <div class="search_right_attr_label">
+                                            <strong>Data Models</strong>
+                                        </div>
+                                        <div class="search_right_attr_value">
+                                            ${ getValueLabels(search['dm'], 'data_model_tag') }
+                                        </div>
+                                    </div>
+                                    <div class="search_right_attr">
+                                        <div class="search_right_attr_label">
+                                            <strong>Technologies</strong>
+                                        </div>
+                                        <div class="search_right_attr_value">
+                                            ${ getValueLabels(search['tex']) }
+                                        </div>
+                                    </div>
+                                </div>
+                            </div>`;
+
+            if (search['action.escu.search_type'] === "support") {
+                //Process Support Search Accordion
+
+                let mappings = JSON.parse(search['action.escu.mappings']);
+                $('#search_support').append(searchPanel);
+
+                // Adding extra params to support search
+                let supportLeftAttr = `<div class="search_left_attr">
+                    <div class="search_right_attr_label">
+                        <strong>Explain It Like I'm 5</strong>
+                    </div>
+                    <div class="search_left_attr_value">
+                        ${ converter.makeHtml(search['action.escu.eli5']) }
+                    </div>
+                </div>`;
+
+
+                $(`#${searchSelector}-eli5`).append(supportLeftAttr);
+
+            } else if (search['action.escu.search_type'] === "detection") {
+                let mappings = JSON.parse(search['action.escu.mappings']);
+                $('#search_detection').append(searchPanel);
+                // Adding extra params to detection search
+                let detectionAttrTop = `
+                <div class="search_right_attr">
+                    <div class="search_right_attr_label">
+                        <strong>ATT&CK</strong>
+                    </div>
+                    <div class="search_right_attr_value">
+                        ${ getValueLabels(mappings.mitre_attack, 'attack_tag') }
+                    </div>
+                </div>
+                <div class="search_right_attr">
+                    <div class="search_right_attr_label">
+                      <strong>Kill Chain Phases</strong>
+                    </div>
+                    <div class="search_right_attr_value">
+                        ${ getValueLabels(mappings.kill_chain_phases, 'kill_chain_tag') }
+                    </div>
+                </div>
+                <div class="search_right_attr">
+                    <div class="search_right_attr_label">
+                        <strong>CIS Controls</strong>
+                    </div>
+                    <div class="search_right_attr_value">
+                        ${ getValueLabels(mappings.cis20) }
+                    </div>
+                </div>
+                `;
+
+                let detectionAttrBottom = `
+                <div class="search_right_attr">
+                    <div class="search_right_attr_label">
+                        <strong>Asset at Risk</strong>
+                    </div>
+                    <div class="search_right_attr_value">
+                        ${ search['action.escu.asset_at_risk'] }
+                    </div>
+                </div>
+                <div class="search_right_attr">
+                    <div class="search_right_attr_label">
+                        <strong>Confidence</strong>
+                    </div>
+                    <div class="search_right_attr_value">
+                        ${ search['action.escu.confidence'] }
+                    </div>
+                </div>
+                 <div class="search_right_attr">
+                    <div class="search_right_attr_label">
+                        <strong>Creation Date</strong>
+                    </div>
+                    <div class="search_right_attr_value">
+                        ${ search['action.escu.creation_date'] }
+                    </div>
+                </div>
+                 <div class="search_right_attr">
+                    <div class="search_right_attr_label">
+                        <strong>Modification Date</strong>
+                    </div>
+                    <div class="search_right_attr_value">
+                        ${ search['action.escu.modification_date'] }
+                    </div>
+                </div>`;
+
+                let detectionLeftAttr = `<div class="search_left_attr">
+                        <div class="search_right_attr_label">
+                            <strong>Explain It Like I'm 5</strong>
+                        </div>
+                        <div class="search_left_attr_value">
+                            ${ converter.makeHtml(search['action.escu.eli5']) }
+                        </div>
+                    </div>`;
+                $(detectionAttrTop).insertBefore($(`#${searchSelector}-content`).find('.data_model_srch_attr'));
+                $(`#${searchSelector}-content`).find('.search_right_panel').append(detectionAttrBottom);
+                $(`#${searchSelector}-eli5`).append(detectionLeftAttr);
+            } else if (search['action.escu.search_type'] === "contextual") {
+                //Process contextual Search Accordion
+
+                let mappings = JSON.parse(search['action.escu.mappings']);
+                $('#search_contextual').append(searchPanel);
+
+
+                // Adding extra params to contextual search
+                let contextualLeftAttr = `<div class="search_left_attr">
+                    <div class="search_right_attr_label">
+                        <strong>Explain It Like I'm 5</strong>
+                    </div>
+                    <div class="search_left_attr_value">
+                        ${ converter.makeHtml(search['action.escu.eli5']) }
+                    </div>
+                </div>`;
+                $(`#${searchSelector}-eli5`).append(contextualLeftAttr);
+
+            } else if (search['action.escu.search_type'] === "investigative") {
+                //Process Investigative Search Accordion
+                let mappings = JSON.parse(search['action.escu.mappings']);
+                $('#search_investigative').append(searchPanel);
+
+                // Adding extra params to investigative search
+                let investigativeLeftAttr = `<div class="search_left_attr">
+                    <div class="search_right_attr_label">
+                        <strong>Explain It Like I'm 5</strong>
+                    </div>
+                    <div class="search_left_attr_value">
+                        ${ converter.makeHtml(search['action.escu.eli5']) }
+                    </div>
+                </div>`;
+
+
+                $(`#${searchSelector}-eli5`).append(investigativeLeftAttr);
+            }
+
+            /*
+            let updatedAttr = `
+            <div class="search_right_attr">
+                <div class="search_right_attr_label">
+                    <strong>Last Updated</strong>
+                </div>
+                <div class="search_right_attr_value">
+                    ${ search['updated'] }
+                </div>
+            </div>
+            `;
+            $(`#${searchSelector}-content`).find('.search_right_panel').append(updatedAttr);
+            */
+
+            $(`#${ btnID }`).on('click', (evt) => {
+                console.log($(evt.target).data("searchType"));
+                if ($(evt.target).data("searchType") === "detection") {
+                    splunkUtil.redirect_to('app/SplunkEnterpriseSecuritySuite/correlation_search_edit', {
+                        search: `${$(evt.target).data("searchName")}`
+                    }, window.open(), true);
+                } else {
+                    splunkUtil.redirect_to(`manager/DA-ESS-ContentUpdate/saved/searches`, {
+                        search: `${$(evt.target).data("searchName")}`
+                    }, window.open(), true);
+                }
+            })
+
+            let searchManagerID = search['action.escu.full_search_name'].split(' ').join('');
+
+            let searchManager = new SearchManager({
+                id: searchManagerID,
+                earliest_time: "-24h@h",
+                latest_time: "now",
+                status_buckets: 300,
+                required_field_list: "*",
+                preview: true,
+                cache: true,
+                autostart: false, // Prevent the search from running automatically
+                search: search['search'],
+            });
+
+            let searchBar = new SearchBarView({
+                id: searchID,
+                managerId: searchManagerID,
+                timerange: true,
+                el: $('.' + controlID),
+                value: search['search'],
+                timerange_preset: "Last 24 hours"
+            }).render();
+
+            let tableviewer = new TableView({
+                id: resultsControlID,
+                managerid: searchManagerID,
+                pageSize: 5,
+                el: $("." + resultsControlID)
+            }).render();
+
+            searchBar.on("change", function() {
+                searchManager.settings.unset("search");
+
+                // Update the search query
+                searchManager.settings.set("search", searchBar.val());
+
+                // Run the search (because autostart=false)
+                searchManager.startSearch();
+            });
+
+            searchBar.timerange.on("change", function() {
+                // Update the time range of the search
+                searchManager.search.set(searchBar.timerange.val());
+
+                // Run the search (because autostart=false)
+                searchManager.startSearch();
+            })
+
+
+            renderedComponents.push(searchID, searchManagerID, resultsControlID);
+
+        });
+
+        $('#accordion').accordion({
+            heightStyle: "content"
+        });
+        $('#search_support').accordion({
+            heightStyle: "content"
+        });
+        $('#search_detection').accordion({
+            heightStyle: "content"
+        });
+        $('#search_contextual').accordion({
+            heightStyle: "content"
+        });
+        $('#search_investigative').accordion({
+            heightStyle: "content"
+        });
+    }
+
+    function clearSearchView() {
+        if ($('#accordion').hasClass('ui-accordion')) {
+            $('#accordion').accordion('destroy');
+        }
+
+        if ($('#search_support').hasClass('ui-accordion')) {
+            $('#search_support').accordion('destroy');
+            $('#search_support').empty();
+        }
+
+        if ($('#search_detection').hasClass('ui-accordion')) {
+            $('#search_detection').accordion('destroy');
+            $('#search_detection').empty();
+        }
+
+        if ($('#search_contextual').hasClass('ui-accordion')) {
+            $('#search_contextual').accordion('destroy');
+            $('#search_contextual').empty();
+        }
+
+        if ($('#search_investigative').hasClass('ui-accordion')) {
+            $('#search_investigative').accordion('destroy');
+            $('#search_investigative').empty();
+        }
+
+        $('.configure_in_es').unbind("click");
+
+        let len = renderedComponents.length;
+        while (len--) {
+            let id = renderedComponents.pop();
+            mvc.Components.getInstance(id).dispose();
+        }
+    }
+
+    function getValueLabels(values, className) {
+        let cls = "";
+        if (className !== undefined || className) {
+            cls = className;
+        }
+        let valueArray = [];
+        if (values) {
+            if (typeof values === "string") {
+                valueArray.push(values)
+            } else {
+                valueArray = values;
+            }
+        }
+        let htmlTmpl = "";
+        valueArray.forEach(val => {
+            htmlTmpl += `<div class="value_label ${ cls }">${ val }</div>&nbsp;`
+        });
+
+        return htmlTmpl;
+    }
+});

package/appserver/static/escu_summary.css

@@ -0,0 +1,66 @@
+.btn-pill  {
+    display: inline;
+}
+
+.killchain-phases {
+    width: 100%;
+    margin-left: 6%;
+}
+
+.killchain_card {
+    width: 14%;
+    float:left;
+}
+
+.killchain {
+    height: 55px;
+    line-height: 55px;
+    -webkit-clip-path: polygon(75% 0%, 100% 50%, 75% 100%, 0% 100%, 25% 50%, 0% 0%);
+    clip-path: polygon(75% 0%, 100% 50%, 75% 100%, 0% 100%, 25% 50%, 0% 0%);
+}
+
+.killchain-text {
+    font-size: 300%;
+    font-weight: 600;
+    color: #fff;
+    text-align: center;
+    white-space: wrap;
+}
+
+
+.killchain-label {
+    margin-top: 7px;
+    text-align: center;
+    color: #666;
+    font-size: 14px;
+    font-weight: 700;
+    margin-left: 10%;
+    width: 60%;
+}
+
+.killchain-text-one-line {
+    font-size: 16px;
+    color: #fff;
+    font-family: system-ui;
+    white-space: wrap;
+    width: 100%;
+    padding-top: 7%;
+    padding-left: 26%;
+}
+
+.killchain-text-second {
+    font-size: 16px;
+    color: #fff;
+    font-family: system-ui;
+    margin-top: -10px;
+    padding-left: 26%;
+}
+
+.killchain-phases {
+    display: inline-block;
+    width: 100%;
+}
+
+.notfirst {
+    margin-left: -2%;
+}

package/appserver/static/escu_summary.js

@@ -0,0 +1,78 @@
+require([
+   'underscore',
+   'jquery',
+   'splunkjs/mvc',
+   'splunkjs/mvc/searchmanager',
+   '../app/DA-ESS-ContentUpdate/js/lib/tabs',
+   'css!../app/DA-ESS-ContentUpdate/js/lib/tabs.css',
+   'css!../app/DA-ESS-ContentUpdate/escu_summary.css',
+   'splunkjs/mvc/simplexml/ready!'
+ ], function(_, $, mvc, SearchManager) {
+    $('.es-soc-analytic-story-stats').html(_.template('<%- _("Analytic Story Summary").t() %>'));
+    $('.es-soc-search-stats').html(_.template('<%- _("Search Summary").t() %>'));
+
+    const tokenModel = mvc.Components.get('default');
+    const submittedTokens = mvc.Components.get('submitted');
+
+    $.ajax({
+        url: Splunk.util.make_url('/splunkd/__raw/servicesNS/nobody/DA-ESS-ContentUpdate/apps/local'),
+        type: 'GET',
+        async: true,
+        data: {
+          output_mode: 'json',
+          count: -1,
+        },
+    }).done(result => {
+       if (result.entry) {
+        const foundEss = result.entry.find(app => app.name === 'SplunkEnterpriseSecuritySuite');
+        if (foundEss.content.version === "5.2.0") {
+          submittedTokens.set('explore-use-case-es-show', 'true');
+          const use_case_library_link = Splunk.util.make_url('app/SplunkEnterpriseSecuritySuite/ess_use_case_library');
+          const template = `<div class="alert alert-info"><i class="icon-alert" />
+                ${ _('Εxplore ESCU content updates directly from the Use Case Library within ES. To explore it, click').t() }
+                <a href="<%- use_case_library_link %>"> ${ _('here').t() }</a>.
+              </div>`;
+          $('#explore-use-case-es-info').html(_.template(template, { use_case_library_link: use_case_library_link }));
+        }
+      }
+    }).fail(err => {
+    });
+
+
+
+    // searchQuery -
+    let kcpSearch = new SearchManager({
+      id: "kcpSearch",
+      preview: true,
+      cache: true,
+      status_buckets: 300,
+      earliest_time: '-24h@h',
+      latest_time: 'now',
+      search: '| rest /services/configs/conf-analytic_stories splunk_server=local count=0 | spath input=mappings path=kill_chain_phases{} output=kcp | stats count by kcp',
+    });
+
+    let results = kcpSearch.data("preview");
+
+    results.on("data", function() {
+      results.data().rows.forEach(row => {
+        let killchainID = '#' + row[0].toLowerCase().replace(/ /g,'');
+        $(killchainID).html(row[1]);
+      });
+    });
+
+    $('#analytic_filter_clear').on('click', function() {
+      tokenModel.set('form.as_cis', '*');
+      tokenModel.set('form.as_category', '*');
+      tokenModel.set('form.as_kill_chain_phase', '*');
+      tokenModel.set('form.as_mitre_attack', '*');
+      tokenModel.set('form.as_data_models', '*');
+    });
+
+    $('#search_filter_clear').on('click', function() {
+      tokenModel.set('form.cis', '*');
+      tokenModel.set('form.searchtype', '*');
+      tokenModel.set('form.kill_chain_phase', '*');
+      tokenModel.set('form.mitre_attack', '*');
+      tokenModel.set('form.data_models', '*');
+    });
+ });

package/appserver/static/feedback.js

@@ -0,0 +1,19 @@
+
+ require([
+   'underscore',
+   'jquery',
+   'splunkjs/mvc',
+   'splunkjs/mvc/simplexml/ready!'
+ ], function(_, $, mvc, TableView) {
+
+   var defaultTokenSpace = mvc.Components.getInstance('default');
+
+   // This will take every textarea that has a data-token attribute and will make the given token with the value of the textarea
+   $('textarea[data-token]').each(function (textarea) {
+     $(this).on('input', function(input) {
+         var token_to_set = $(this).data('token');
+        defaultTokenSpace.set(token_to_set, $(this).val());
+     })
+   })
+ });
+

package/bin/da_ess_contentupdate/alert_actions_base.py

@@ -0,0 +1,211 @@
+import csv
+import gzip
+import sys
+
+from splunk.clilib.bundle_paths import make_splunkhome_path
+
+sys.path.insert(0, make_splunkhome_path(["etc", "apps", "Splunk_SA_CIM", "lib"]))
+
+import requests
+from cim_actions import ModularAction
+from logging_helper import get_logger
+import logging
+from splunk_aoblib.rest_helper import TARestHelper
+from splunk_aoblib.setup_util import Setup_Util
+
+
+class ModularAlertBase(ModularAction):
+    def __init__(self, ta_name, alert_name):
+        self._alert_name = alert_name
+        # self._logger_name = "modalert_" + alert_name
+        self._logger_name = alert_name + "_modalert"
+        self._logger = get_logger(self._logger_name)
+        super(ModularAlertBase, self).__init__(
+            sys.stdin.read(), self._logger, alert_name)
+        self.setup_util_module = None
+        self.setup_util = None
+        self.result_handle = None
+        self.ta_name = ta_name
+        self.splunk_uri = self.settings.get('server_uri')
+        self.setup_util = Setup_Util(self.splunk_uri, self.session_key, self._logger)
+
+        level = self.get_log_level()
+        if level:
+            self._logger.setLevel(level)
+        self.rest_helper = TARestHelper(self._logger)
+
+    def log_error(self, msg):
+        self.message(msg, 'failure', level=logging.ERROR)
+
+    def log_info(self, msg):
+        self.message(msg, 'success', level=logging.INFO)
+
+    def log_debug(self, msg):
+        self.message(msg, None, level=logging.DEBUG)
+
+    def log_warn(self, msg):
+        self.message(msg, None, level=logging.WARN)
+
+    def set_log_level(self, level):
+        self._logger.setLevel(level)
+
+    def get_param(self, param_name):
+        return self.configuration.get(param_name)
+
+    def get_global_setting(self, var_name):
+        return self.setup_util.get_customized_setting(var_name)
+
+    def get_user_credential(self, username):
+        '''
+        if the username exists, return
+        {
+            "username": username,
+            "password": credential
+        }
+        '''
+        return self.setup_util.get_credential_by_username(username)
+
+    @property
+    def log_level(self):
+        return self.get_log_level()
+
+    @property
+    def proxy(self):
+        return self.get_proxy()
+
+    def get_log_level(self):
+        return self.setup_util.get_log_level()
+
+    def get_proxy(self):
+        ''' if the proxy setting is set. return a dict like
+        {
+        proxy_url: ... ,
+        proxy_port: ... ,
+        proxy_username: ... ,
+        proxy_password: ... ,
+        proxy_type: ... ,
+        proxy_rdns: ...
+        }
+        '''
+        return self.setup_util.get_proxy_settings()
+
+    def _get_proxy_uri(self):
+        uri = None
+        proxy = self.get_proxy()
+        if proxy and proxy.get('proxy_url') and proxy.get('proxy_type'):
+            uri = proxy['proxy_url']
+            if proxy.get('proxy_port'):
+                uri = '{0}:{1}'.format(uri, proxy.get('proxy_port'))
+            if proxy.get('proxy_username') and proxy.get('proxy_password'):
+                uri = '{0}://{1}:{2}@{3}/'.format(proxy['proxy_type'], proxy[
+                                                  'proxy_username'], proxy['proxy_password'], uri)
+            else:
+                uri = '{0}://{1}'.format(proxy['proxy_type'], uri)
+        return uri
+
+    def send_http_request(self, url, method, parameters=None, payload=None, headers=None, cookies=None, verify=True, cert=None, timeout=None, use_proxy=True):
+        return self.rest_helper.send_http_request(url=url, method=method, parameters=parameters, payload=payload,
+                                                  headers=headers, cookies=cookies, verify=verify, cert=cert,
+                                                  timeout=timeout,
+                                                  proxy_uri=self._get_proxy_uri() if use_proxy else None)
+
+    def build_http_connection(self, config, timeout=120,
+                              disable_ssl_validation=False):
+        from httplib2 import (socks, ProxyInfo, Http)
+        """
+        :config: dict like, proxy and account information are in the following
+                format {
+                    "username": xx,
+                    "password": yy,
+                    "proxy_url": zz,
+                    "proxy_port": aa,
+                    "proxy_username": bb,
+                    "proxy_password": cc,
+                    "proxy_type": http,http_no_tunnel,sock4,sock5,
+                    "proxy_rdns": 0 or 1,
+                }
+        :return: Http2.Http object
+        """
+        if not config:
+            config = {}
+
+        proxy_type_to_code = {
+            "http": socks.PROXY_TYPE_HTTP,
+            "http_no_tunnel": socks.PROXY_TYPE_HTTP_NO_TUNNEL,
+            "socks4": socks.PROXY_TYPE_SOCKS4,
+            "socks5": socks.PROXY_TYPE_SOCKS5,
+        }
+        if config.get("proxy_type") in proxy_type_to_code:
+            proxy_type = proxy_type_to_code[config["proxy_type"]]
+        else:
+            proxy_type = socks.PROXY_TYPE_HTTP
+
+        rdns = config.get("proxy_rdns")
+
+        proxy_info = None
+        if config.get("proxy_url") and config.get("proxy_port"):
+            if config.get("proxy_username") and config.get("proxy_password"):
+                proxy_info = ProxyInfo(proxy_type=proxy_type,
+                                       proxy_host=config["proxy_url"],
+                                       proxy_port=int(config["proxy_port"]),
+                                       proxy_user=config["proxy_username"],
+                                       proxy_pass=config["proxy_password"],
+                                       proxy_rdns=rdns)
+            else:
+                proxy_info = ProxyInfo(proxy_type=proxy_type,
+                                       proxy_host=config["proxy_url"],
+                                       proxy_port=int(config["proxy_port"]),
+                                       proxy_rdns=rdns)
+        if proxy_info:
+            http = Http(proxy_info=proxy_info, timeout=timeout,
+                        disable_ssl_certificate_validation=disable_ssl_validation)
+        else:
+            http = Http(timeout=timeout,
+                        disable_ssl_certificate_validation=disable_ssl_validation)
+
+        if config.get("username") and config.get("password"):
+            http.add_credentials(config["username"], config["password"])
+        return http
+
+    def process_event(self, *args, **kwargs):
+        raise NotImplemented()
+
+    def pre_handle(self, num, result):
+        result.setdefault('rid', str(num))
+        self.update(result)
+        return result
+
+    def get_events(self):
+        self.result_handle = gzip.open(self.results_file, 'rb')
+        return (self.pre_handle(num, result) for num, result in enumerate(csv.DictReader(self.result_handle)))
+
+    def prepare_meta_for_cam(self):
+        with gzip.open(self.results_file, 'rb') as rf:
+            for num, result in enumerate(csv.DictReader(rf)):
+                result.setdefault('rid', str(num))
+                self.update(result)
+                self.invoke()
+                break
+
+    def run(self, argv):
+        status = 0
+        if len(argv) < 2 or argv[1] != "--execute":
+            msg = 'Error: argv="{}", expected="--execute"'.format(argv)
+            print >> sys.stderr, msg
+            sys.exit(1)
+
+        try:
+            status = self.process_event()
+        except Exception as e:
+            msg = "Unexpected error: {}."
+            if e.message:
+                self.log_error(msg.format(e.message))
+            else:
+                import traceback
+                self.log_error(msg.format(traceback.format_exc()))
+            sys.exit(2)
+        finally:
+            if self.result_handle:
+                self.result_handle.close()
+
+        return status

package/bin/da_ess_contentupdate/cim_actions.py

@@ -0,0 +1,508 @@
+import collections
+import csv
+import json
+import logging
+import logging.handlers
+import os
+import random
+import re
+import splunk.rest as rest
+import time
+
+from splunk.clilib.bundle_paths import make_splunkhome_path
+from splunk.util                import mktimegm, normalizeBoolean
+
+# set the maximum allowable CSV field size
+#
+# The default of the csv module is 128KB; upping to 10MB. See SPL-12117 for
+# the background on issues surrounding field sizes.
+# (this method is new in python 2.5)
+csv.field_size_limit(10485760)
+
+
+class InvalidResultID(Exception):
+    pass
+
+
+class ModularAction(object):
+    DEFAULT_MSGFIELDS = ['signature',
+                         'action_name',
+                         'search_name',
+                         'sid',
+                         'orig_sid',
+                         'rid',
+                         'orig_rid',
+                         'app',
+                         'user',
+                         'action_mode',
+                         'action_status']
+    DEFAULT_MESSAGE   = 'sendmodaction - ' + ' '.join(['{i}="{{d[{i}]}}"'.format(i=i) for i in DEFAULT_MSGFIELDS])
+    # The above yields a string.format() compatible format string:
+    #
+    #   'sendmodaction - signature="{d[signature]}" action_name="{d[action_name]}"
+    #   search_name="{d[search_name]}" sid="{d[sid]}" orig_sid="{d[orig_sid]}"
+    #   rid="{d[rid]}" orig_rid="{d[orig_rid]}" app="{d[app]}" user="{d[user]}"
+    #   action_mode="{d[action_mode]}" action_status="{d[action_status]}"'
+
+    DEFAULT_DROPEXP  = lambda x: ((x.startswith('_') and x not in ['_raw', '_time'])
+                                  or x.startswith('date_')
+                                  or x in ['punct', 'sid', 'rid', 'orig_sid', 'orig_rid'])
+
+    DEFAULT_MAPEXP   = lambda x: (x.startswith('tag::')
+                                  or x in ['_time', '_raw', 'splunk_server', 'index',
+                                           'source', 'sourcetype', 'host', 'linecount',
+                                           'timestartpos', 'timeendpos', 'eventtype',
+                                           'tag', 'search_name', 'event_hash', 'event_id'])
+
+    DEFAULT_HEADER   = '***SPLUNK*** index="%s" host="%s" source="%s"'
+    DEFAULT_BREAKER  = '==##~~##~~  1E8N3D4E6V5E7N2T9 ~~##~~##==\n'
+    DEFAULT_IDLINE   = '***Common Action Model*** orig_action_name="%s" orig_sid="%s" orig_rid="%s" sourcetype="%s"\n'
+    DEFAULT_INDEX    = 'summary'
+    DEFAULT_CHUNK    = 50000
+
+    SHORT_FORMAT     = '%(asctime)s %(levelname)s %(message)s'
+
+    def __init__(self, settings, logger, action_name='unknown'):
+        """ Initialize ModularAction class.
+
+        @param settings:    A modular action payload in JSON format.
+        @param logger:      A logging instance.
+                            Recommend using ModularAction.setup_logger.
+        @param action_name: The action name.
+                            action_name in payload will take precedence.
+        """
+        self.settings      = json.loads(settings)
+        self.logger        = logger
+        self.session_key   = self.settings.get('session_key')
+        self.sid           = self.settings.get('sid')
+        self.sid_snapshot  = ''
+        ## if sid contains rt_scheduler with snapshot-sid; drop snapshot-sid
+        ## sometimes self.sid may be an integer (1465593470.1228)
+        try:
+            rtsid    = re.match('^(rt_scheduler.*)\.(\d+)$', self.sid)
+            if rtsid:
+                self.sid          = rtsid.group(1)
+                self.sid_snapshot = rtsid.group(2)
+        except:
+            pass
+
+        ## rid_ntuple is a named tuple that represents
+        ## the three variables that change on a per-result basis
+        self.rid_ntuple    = collections.namedtuple('ID', ['orig_sid','rid','orig_rid'])
+        ## rids is a list of rid_ntuple values
+        ## automatically maintained by update() calls
+        self.rids          = []
+        ## current orig_sid based on update()
+        ## aka self.rids[-1].orig_sid
+        self.orig_sid      = ''
+        ## current rid based on update()
+        ## aka self.rids[-1].rid
+        self.rid           = ''
+        ## current orig_rid based on update()
+        ## aka self.rids[-1].orig_rid
+        self.orig_rid      = ''
+
+        self.results_file  = self.settings.get('results_file')
+        ## info
+        self.info          = {}
+        if self.results_file:
+            self.info_file = os.path.join(os.path.dirname(self.results_file), 'info.csv')
+        self.search_name   = self.settings.get('search_name')
+        self.app           = self.settings.get('app')
+        self.user          = self.settings.get('user') or self.settings.get('owner')
+        self.configuration = self.settings.get('configuration', {})
+        ## enforce configuration is a 'dict'
+        if not isinstance(self.configuration, dict):
+            self.configuration = {}
+        ## set loglevel to DEBUG if verbose
+        if normalizeBoolean(self.configuration.get('verbose', 'false')):
+            self.logger.setLevel(logging.DEBUG)
+            self.logger.debug('loglevel set to DEBUG')
+        ## use | sendalert param.action_name=$action_name$
+        self.action_name = self.configuration.get('action_name') or action_name
+        ## use sid to determine action_mode
+        if isinstance(self.sid, basestring) and 'scheduler' in self.sid:
+            self.action_mode = 'saved'
+        else:
+            self.action_mode = 'adhoc'
+
+        self.action_status = ''
+        ## Since we don't use the result object we get from settings it will be purged
+        try:
+            del self.settings['result']
+        except Exception:
+            pass
+        ## events
+        self.events = []
+
+    def addinfo(self):
+        """ The purpose of this method is to populate the
+        modular action info variable with the contents of info.csv.
+
+        @raise Exception: raises Exception if self.info_file could not be opened
+                          or if there were problems parsing the info.csv data
+        """
+        if self.info_file:
+            try:
+                with open(self.info_file, 'rU') as fh:
+                    self.info = csv.DictReader(fh).next()
+            except Exception as e:
+                self.message('Could not retrieve info.csv', level=logging.WARN)
+
+    def addjobinfo(self):
+        """ The purpose of this method is to populate the job variable
+        with the contents from REST (/services/search/jobs/<sid>)
+
+        SPL-112815 - sendalert - not all $job.<param>$ parameters come through
+
+        @raise Exception: raises Exception if search job information could not
+                          be retrieved via REST (search/jobs) based on self.sid
+        """
+        self.job = {}
+        if self.sid:
+            try:
+                response, content = rest.simpleRequest('search/jobs/%s' % self.sid,
+                                        sessionKey=self.session_key,
+                                        getargs={'output_mode': 'json'})
+                if response.status == 200:
+                    self.job = json.loads(content)['entry'][0]['content']
+                    self.message('Successfully retrieved search job info')
+                    self.logger.debug(self.job)
+                else:
+                    self.message('Could not retrieve search job info', level=logging.WARN)
+            except Exception as e:
+                self.message('Could not retrieve search job info', level=logging.WARN)
+
+    def message(self, signature, status=None, rids=None, level=logging.INFO, **kwargs):
+        """ The purpose of this method is to provide a common messaging interface.
+
+        @param signature: A string representing the message we want to log.
+        @param status:    An optional status that we want to log.
+                          Defaults to None.
+        @param rids:      An optional list of rid_ntuple values in case we
+                          want to generate the message for multiple rids.
+                          Defaults to None (use the rid currently loaded).
+        @param level:     The logging level to use when writing the message.
+                          Defaults to logging.INFO (INFO)
+        @param kwargs:    Additional keyword arguments to be included with the
+                          message.
+                          Defaults to "no arguments".
+
+        @return message:  This method logs the message; however, for
+                          backwards compatibility we also return the message.
+        """
+        ## status
+        status = status or self.action_status or ''
+        ## rid
+        if not isinstance(rids, list):
+            rids = [self.rid_ntuple(self.orig_sid, self.rid, self.orig_rid)]
+        ## kwargs - prune any duplicate keys based on DEFAULT_MSGFIELDS
+        ##          prune any keys with special characters [A-Za-z_]+
+        newargs = [x for x in kwargs\
+            if (x not in ModularAction.DEFAULT_MSGFIELDS) and re.match('[A-Za-z_]+', x)]
+        ## MSG
+        msg = '%s %s' % (ModularAction.DEFAULT_MESSAGE, ' '.join(['{i}="{{d[{i}]}}"'.format(i=i) for i in newargs]))
+
+        # This will set the default value of any value NOT in the dictionary to the
+        # empty string.
+        argsdict = collections.defaultdict(str)
+        # order is important here - here we update first from kwargs, then from our
+        # expected arg set.
+        argsdict.update(kwargs)
+        argsdict.update({
+            'signature':     signature or '',
+            'action_name':   self.action_name or '',
+            'search_name':   self.search_name or '',
+            'sid':           self.sid or '',
+            'app':           self.app or '',
+            'user':          self.user or '',
+            'action_mode':   self.action_mode or '',
+            'action_status': status
+        })
+
+        for rid_ntuple in rids:
+            if len(rid_ntuple)==3:
+                ## Update the arguments dictionary
+                argsdict.update({
+                    'orig_sid': rid_ntuple.orig_sid or '',
+                    'rid':      rid_ntuple.rid or '',
+                    'orig_rid': rid_ntuple.orig_rid or ''
+                })
+                ## This is where the magic happens. The format string will use the
+                ## attributes of "argsdict"
+                message = msg.format(d=argsdict)
+                ## prune empty string key-value pairs
+                for match in re.finditer('[A-Za-z_]+=\"\"(\s|$)', message):
+                    message = message.replace(match.group(0),'',1)
+                message = message.strip()
+                self.logger.log(level, message)
+            else:
+                self.logger.warn('Could not unpack rid_ntuple')
+                message = ''
+
+        return message
+
+    def update(self, result):
+        """ The purpose of this method is to update the ModularAction instance
+        identifiers based on the current result being operated on.
+
+        This is the most important method in the library as it sets up
+        rid, orig_sid, and orig_rid to be used by subsequent class methods.
+
+        Not calling update() immediately for each result before doing additional
+        work can have adverse affects.
+
+        @param signature: A string representing the message we want to log.
+        @param status:    An optional status that we want to log.
+                          Defaults to None.
+        @param rids:      An optional list of rid_ntuple values in case we
+                          want to generate the message for multiple rids.
+                          Defaults to None (use the rid currently loaded).
+        @param level:     The logging level to use when writing the message.
+                          Defaults to logging.INFO (INFO)
+        @param kwargs:    Additional keyword arguments to be included with the
+                          message.
+                          Defaults to "no arguments".
+
+        @return message:  This method logs the message; however, for
+                          backwards compatiblity we also return the message.
+        """
+        ## This is for events/results that were created as the result of a previous action
+        self.orig_sid = result.get('orig_sid', '')
+        ## This is for events/results that were created as the result of a previous action
+        self.orig_rid = result.get('orig_rid', '')
+        if 'rid' in result and isinstance(result['rid'], (basestring, int)):
+            self.rid = str(result['rid'])
+            if self.sid_snapshot:
+                self.rid = '%s.%s' % (self.rid, self.sid_snapshot)
+            ## add result info to list of named tuples
+            self.rids.append(self.rid_ntuple(self.orig_sid, self.rid, self.orig_rid))
+        else:
+            raise InvalidResultID('Result must have an ID')
+
+    def invoke(self):
+        """ The purpose of this method is to generate per-result invocation messages.
+        This method is used to identify that an action is being attempted on a per-result basis.
+
+        Remember to call update() prior to invoke() to ensure that the invocation message
+        reflects the appropriate identifiers.
+        """
+        self.message('Invoking modular action')
+
+    def result2stash(self, result, dropexp=DEFAULT_DROPEXP, mapexp=DEFAULT_MAPEXP, addinfo=False):
+        """ The purpose of this method is to formulate an event in stash format
+
+        @param result:  The result dictionary to generate a stash event for.
+        @param dropexp: A lambda expression used to determine whether a field
+                        should be dropped or not.
+                        Defaults to DEFAULT_DROPEXP.
+        @param mapexp:  A lambda expression used to determine whether a field
+                        should be mapped (prepended with "orig_") or not.
+                        Defaults to DEFAULT_MAPEXP.
+        @param addinfo: Whether or not to add search information to the event.
+                        "info" includes search_now, info_min_time, info_max_time,
+                        and info_search_time fields.
+                        Requires that information was loaded into the ModularAction
+                        instance via addinfo()
+
+        @return _raw:   Returns a string which represents the result in stash format.
+
+        The following example has been broken onto multiple lines for readability:
+        06/21/2016 10:00:00 -0700,
+        search_name="Access - Brute Force Access Behavior Detected - Rule",
+        search_now=0.000, info_min_time=1466528400.000, info_max_time=1466532600.000, info_search_time=1465296264.179,
+        key1=key1val, key2=key2val, key3=key3val, key4=key4val1, key4=key4val2, ...
+        """
+        dropexp      = dropexp or (lambda x: False)
+        mapexp       = mapexp  or (lambda x: False)
+        orig_dropexp = lambda x: x.startswith('orig_') and x[5:] in result and mapexp(x[5:])
+
+        ## addinfo
+        if addinfo:
+            result['info_min_time']    = self.info.get('_search_et', '0.000')
+            info_max_time              = self.info.get('_search_lt')
+            if not info_max_time or info_max_time==0 or info_max_time=='0':
+                info_max_time = '+Infinity'
+            result['info_max_time']    = info_max_time
+            result['info_search_time'] = self.info.get('_timestamp', '')
+
+        ## construct _raw
+        _raw = '%s' % result.get('_time', mktimegm(time.gmtime()))
+        if self.search_name:
+            _raw += ', search_name="%s"' % self.search_name
+
+        processed_keys = []
+        for key, val in sorted(result.items()):
+            vals = []
+            ## if we have a proper mv field
+            if (key.startswith('__mv_')
+                and val and isinstance(val, basestring)
+                and val.startswith('$') and val.endswith('$')):
+                real_key = key[5:]
+                vals     = val[1:-1].split('$;$')
+            ## if proper sv field
+            elif val and not key.startswith('__mv_'):
+                real_key = key
+                vals     = [val]
+
+            ## if we have vals and key hasn't been processed
+            ## and key is not to be dropped...
+            if (vals
+                and (real_key not in processed_keys)
+                and not dropexp(real_key)
+                and not orig_dropexp(real_key)):
+                ## iterate vals
+                for val in vals:
+                    ## format literal '$'
+                    if key.startswith('__mv'):
+                        val = val.replace('$$', '$')
+                    ## escape quotes
+                    if isinstance(val, basestring):
+                        val = val.replace('"', r'\"')
+                    ## check map
+                    if mapexp(real_key):
+                        _raw += ', %s="%s"' % ('orig_' + real_key.lstrip('_'), val)
+                    else:
+                        _raw += ', %s="%s"' % (real_key, val)
+                processed_keys.append(real_key)
+
+        return _raw
+
+    def addevent(self, raw, sourcetype, cam_header=True):
+        """ The purpose of this method is to add a properly constructed event
+        to the events list in the ModularAction instance.  This ensures events
+        are created with the appropriate index-time header.
+
+        The index-time header is responsible for setting sourcetype,
+        orig_action_name, orig_sid, and orig_rid.  The index-time header will
+        not be present in the _raw of generated events.
+
+        Remember to call update() prior to addevent() to ensure that the events
+        reflect the appropriate orig_sid and orig_rid identifiers.
+
+        @param raw:        The text of the event you want to generate.
+        @param sourcetype: The sourcetype of the event you want to generate.
+        @param cam_header: Optionally exclude the inclusion of the index-time header.
+                           Defaults to True (include header).
+        """
+        if cam_header:
+            if self.orig_sid:
+                action_idline = ModularAction.DEFAULT_IDLINE % (
+                                    self.action_name,
+                                    self.orig_sid,
+                                    self.orig_rid,
+                                    sourcetype)
+            else:
+                action_idline = ModularAction.DEFAULT_IDLINE % (
+                                    self.action_name,
+                                    self.sid,
+                                    self.rid,
+                                    sourcetype)
+            self.events.append(action_idline + raw)
+        else:
+            self.events.append(raw)
+
+    def writeevents(self, index='summary', host=None, source=None, fext='common_action_model'):
+        """ The purpose of this method is to create arbitrary splunk events
+        from the list of events in the ModularAction instance.
+
+        Please use addevent() for populating the list of events in
+        the ModularAction instance.
+
+        @param index:  The index to write the events to.
+                       Defaults to "summary".
+        @param host:   The value of host the events should take on.
+                       Defaults to None (auto).
+        @param source: The value of source the events should take on.
+                       Defaults to None (auto).
+        @param fext:   The extension of the file to write out.
+                       Files are written to $SPLUNK_HOME/var/spool/splunk.
+                       File extensions can only contain word characters,
+                       dash, and have a 200 char max.
+                       "stash_" is automatically prepended to all extensions.
+                       Defaults to "common_action_model" ("stash_common_action_model").
+                       Only override if you've set up a corresponding props.conf
+                       stanza to handle the extension.
+
+        @return bool:  Returns True if all events were successfully written
+                       Returns False if any errors were encountered
+        """
+        ## internal makeevents method for normalizing strings
+        ## that will be used in the various headers we write out
+        def get_string(input, default):
+            try:
+                return input.replace('"', '_')
+            except AttributeError:
+                return default
+
+        if self.events:
+            ## sanitize file extension
+            if not fext or not re.match('^[\w-]+$', fext):
+                self.logger.warn('Requested file extension was ignored due to invalid characters')
+                fext = 'common_action_model'
+            elif len(fext)>200:
+                self.logger.warn('Requested file extension was ignored due to length')
+                fext = 'common_action_model'
+            ## header
+            header_line = ModularAction.DEFAULT_HEADER % (
+                            get_string(index, ModularAction.DEFAULT_INDEX),
+                            get_string(host, ''),
+                            get_string(source, ''))
+            ## process event chunks
+            for chunk in (self.events[x:x+ModularAction.DEFAULT_CHUNK]
+                          for x in xrange(0, len(self.events), ModularAction.DEFAULT_CHUNK)):
+                ## initialize output string
+                default_breaker = '\n' + ModularAction.DEFAULT_BREAKER
+                fout            = header_line + default_breaker + (default_breaker).join(chunk)
+                ## write output string
+                try:
+                    fn = '%s_%s.stash_%s' % (mktimegm(time.gmtime()), random.randint(0, 100000), fext)
+                    fp = make_splunkhome_path(['var', 'spool', 'splunk', fn])
+                    ## obtain fh
+                    with open(fp, 'w') as fh:
+                        fh.write(fout)
+                except:
+                    signature = 'Error obtaining file handle during makeevents'
+                    self.message(signature, level=logging.ERROR, file_path=fp)
+                    self.logger.exception(signature + ' file_path=%s' % fp)
+                    return False
+            self.message('Successfully created splunk events', event_count=len(self.events))
+            return True
+        return False
+
+    def dowork(self):
+        """ This method serves as an illustration stub.
+        Serves as a container for operations which satisfy the nature of the action.
+        For instance, the third party API call.
+
+        For cleanliness it is recommended that you subclass ModularAction
+        and implement your own dowork() method.
+        """
+        return
+
+    @staticmethod
+    def setup_logger(name, level=logging.INFO, maxBytes=25000000, backupCount=5, format=SHORT_FORMAT):
+        """ Set up a logging instance.
+
+        @param name:        The log file name.
+                            We recommend "$action_name$_modalert".
+        @param level:       The logging level.
+        @param maxBytes:    The maximum log file size before rollover.
+        @param backupCount: The number of log files to retain.
+
+        @return logger:     Returns an instance of logger
+        """
+        logfile = make_splunkhome_path(['var', 'log', 'splunk', name + '.log'])
+        logger = logging.getLogger(name)
+        logger.setLevel(level)
+        logger.propagate = False  # Prevent the log messages from being duplicated in the python.log file
+
+        # Prevent re-adding handlers to the logger object, which can cause duplicate log lines.
+        handler_exists = any([True for h in logger.handlers if h.baseFilename == logfile])
+        if not handler_exists:
+            file_handler = logging.handlers.RotatingFileHandler(logfile, maxBytes=maxBytes, backupCount=backupCount)
+            formatter = logging.Formatter(format)
+            file_handler.setFormatter(formatter)
+            logger.addHandler(file_handler)
+
+        return logger

package/bin/da_ess_contentupdate/cloudconnectlib/__init__.py

@@ -0,0 +1,10 @@
+"""
+APP Cloud Connect
+"""
+import os
+
+from .common.lib_util import register_cacert_locater
+
+register_cacert_locater(os.path.join(os.path.dirname(__file__), 'core', 'cacerts'))
+
+__version__ = '1.0.1'

package/bin/da_ess_contentupdate/cloudconnectlib/client.py

@@ -0,0 +1,77 @@
+import copy
+import os.path
+import traceback
+
+from .common.log import get_cc_logger
+from .common.util import load_json_file
+from .configuration import get_loader_by_version
+from .core import CloudConnectEngine
+from .core.exceptions import ConfigException
+
+_logger = get_cc_logger()
+
+
+class CloudConnectClient(object):
+    """The client of cloud connect used to start a cloud connect engine instance.
+    """
+
+    def __init__(self, context, config_file, checkpoint_mgr):
+        """
+        Constructs a `CloudConnectClient` with `context` which contains variables
+        to render template in the configuration parsed from file `config_file`.
+        :param context: context to render template.
+        :param config_file: file path for load user passed interface.
+        """
+        self._context = context
+        self._config_file = config_file
+        self._engine = None
+        self._config = None
+        self._checkpoint_mgr = checkpoint_mgr
+
+    def _load_config(self):
+        """Load a JSON based configuration definition from file.
+        :return: A `dict` contains user defined JSON interface.
+        """
+        try:
+            conf = load_json_file(self._config_file)
+        except:
+            raise ConfigException(
+                'Unable to load configuration file %s: %s'
+                % (self._config_file, traceback.format_exc())
+            )
+
+        version = conf.get('meta', {'apiVersion', None}).get('apiVersion', None)
+        if not version:
+            raise ConfigException(
+                'Config meta or api version not present in {}'.format(
+                    self._config_file))
+
+        config_loader, schema_file = get_loader_by_version(version)
+        schema_path = os.path.join(
+            os.path.dirname(__file__), 'configuration', schema_file)
+
+        return config_loader.load(conf, schema_path, self._context)
+
+    def start(self):
+        """
+        Initialize a new `CloudConnectEngine` instance and start it.
+        """
+        try:
+            if self._config is None:
+                self._config = self._load_config()
+
+            self._engine = CloudConnectEngine()
+            self._engine.start(
+                context=copy.deepcopy(self._context),
+                config=self._config,
+                checkpoint_mgr=self._checkpoint_mgr
+            )
+        except Exception as ex:
+            _logger.exception('Error while starting client')
+            raise ex
+
+    def stop(self):
+        """Stop the current cloud connect engine.
+        """
+        if self._engine:
+            self._engine.stop()

package/bin/da_ess_contentupdate/cloudconnectlib/common/lib_util.py

@@ -0,0 +1,54 @@
+import os
+import os.path as op
+import platform
+import sys
+
+import __main__
+
+from ..splunktacollectorlib.common import log as stulog
+
+
+def get_main_file():
+    """Return the running mod input file"""
+    return __main__.__file__
+
+
+def get_app_root_dir():
+    """Return the root dir of app"""
+    return op.dirname(op.dirname(op.abspath(get_main_file())))
+
+
+def get_mod_input_script_name():
+    """Return the name of running mod input"""
+    script_name = os.path.basename(get_main_file())
+    if script_name.lower().endswith('.py'):
+        script_name = script_name[:-3]
+    return script_name
+
+
+def register_module(new_path):
+    """ register_module(new_path): adds a directory to sys.path.
+    Do nothing if it does not exist or if it's already in sys.path.
+    """
+    if not os.path.exists(new_path):
+        return
+
+    new_path = os.path.abspath(new_path)
+    if platform.system() == 'Windows':
+        new_path = new_path.lower()
+    for x in sys.path:
+        x = os.path.abspath(x)
+        if platform.system() == 'Windows':
+            x = x.lower()
+        if new_path in (x, x + os.sep):
+            return
+    sys.path.insert(0, new_path)
+
+
+def register_cacert_locater(cacerts_locater_path):
+    for x in sys.modules:
+        if (x == "httplib2" or x.endswith(".httplib2")) and sys.modules[x] \
+                is not None:
+            stulog.logger.warning("Httplib2 module '{}' is already installed. "
+                                  "The ca_certs_locater may not work".format(x))
+    register_module(cacerts_locater_path)

package/bin/da_ess_contentupdate/cloudconnectlib/common/log.py

@@ -0,0 +1,32 @@
+import logging
+
+from solnlib.pattern import Singleton
+from ..splunktacollectorlib.common import log as stulog
+
+
+class CloudClientLogAdapter(logging.LoggerAdapter):
+    __metaclass__ = Singleton
+
+    def __init__(self, logger=None, extra=None, prefix=""):
+        super(CloudClientLogAdapter, self).__init__(logger, extra)
+        self.cc_prefix = prefix if prefix else ""
+
+    def process(self, msg, kwargs):
+        msg = "{} {}".format(self.cc_prefix, msg)
+        return super(CloudClientLogAdapter, self).process(msg, kwargs)
+
+    def set_level(self, val):
+        self.logger.setLevel(val)
+
+
+_adapter = CloudClientLogAdapter(stulog.logger)
+
+
+def set_cc_logger(logger, logger_prefix=''):
+    global _adapter
+    _adapter.logger = logger
+    _adapter.cc_prefix = logger_prefix or ''
+
+
+def get_cc_logger():
+    return _adapter

package/bin/da_ess_contentupdate/cloudconnectlib/common/util.py

@@ -0,0 +1,48 @@
+import json
+from ..splunktalib.common import util
+from solnlib.modular_input.event import XMLEvent
+
+
+def is_valid_bool(val):
+    """Check whether a string can be convert to bool.
+    :param val: value as string.
+    :return: `True` if value can be convert to bool else `False`.
+    """
+    return util.is_true(val) or util.is_false(val)
+
+
+def is_true(val):
+    return util.is_true(val)
+
+
+def is_valid_port(port):
+    """Check whether a port is valid.
+    :param port: port to check.
+    :return: `True` if port is valid else `False`.
+    """
+    try:
+        return 1 <= int(port) <= 65535
+    except ValueError:
+        return False
+
+
+def load_json_file(file_path):
+    """
+    Load a dict from a JSON file.
+    :param file_path: JSON file path.
+    :return: A `dict` object.
+    """
+    with open(file_path, 'r') as file_pointer:
+        return json.load(file_pointer)
+
+
+def format_events(raw_events, time=None,
+                  index=None, host=None, source=None, sourcetype=None,
+                  stanza=None, unbroken=False, done=False):
+    return XMLEvent.format_events(XMLEvent(data, time=time,
+                                           index=index, host=host,
+                                           source=source,
+                                           sourcetype=sourcetype,
+                                           stanza=stanza, unbroken=unbroken,
+                                           done=done) for data in
+                                  raw_events)

package/bin/da_ess_contentupdate/cloudconnectlib/configuration/__init__.py

@@ -0,0 +1 @@
+from .loader import get_loader_by_version

package/bin/da_ess_contentupdate/cloudconnectlib/configuration/loader.py

@@ -0,0 +1,300 @@
+import logging
+import re
+import traceback
+from abc import abstractmethod
+
+from jsonschema import validate, ValidationError
+from munch import munchify
+from ..common.log import get_cc_logger
+from ..common.util import (
+    load_json_file, is_valid_bool, is_valid_port, is_true
+)
+from ..core.exceptions import ConfigException
+from ..core.ext import lookup_method
+from ..core.models import (
+    BasicAuthorization, Request, Processor,
+    Condition, Task, Checkpoint, IterationMode,
+    DictToken
+)
+
+_logger = get_cc_logger()
+
+_PROXY_TYPES = ['http', 'socks4', 'socks5', 'http_no_tunnel']
+_AUTH_TYPES = {
+    'basic_auth': BasicAuthorization
+}
+
+_LOGGING_LEVELS = {
+    'DEBUG': logging.DEBUG,
+    'INFO': logging.INFO,
+    'WARNING': logging.WARNING,
+    'ERROR': logging.ERROR,
+    'FATAL': logging.FATAL,
+    'CRITICAL': logging.CRITICAL
+}
+
+# FIXME Make this configurable
+_DEFAULT_LOG_LEVEL = 'INFO'
+
+
+class CloudConnectConfigLoader(object):
+    """The Base cloud connect configuration loader"""
+
+    @staticmethod
+    def _get_schema_from_file(schema_file):
+        """ Load JSON based schema definition from schema file path.
+        :return: A `dict` contains schema.
+        """
+        try:
+            return load_json_file(schema_file)
+        except:
+            raise ConfigException(
+                'Cannot load schema from file {}: {}'.format(
+                    schema_file, traceback.format_exc())
+            )
+
+    @abstractmethod
+    def load(self, definition, schema_file, context):
+        pass
+
+
+class CloudConnectConfigLoaderV1(CloudConnectConfigLoader):
+    @staticmethod
+    def _render_from_dict(source, ctx):
+        rendered = DictToken(source).render(ctx)
+
+        return dict((k, v.strip() if isinstance(v, basestring) else v)
+                    for k, v in rendered.iteritems())
+
+    def _load_proxy(self, candidate, variables):
+        """
+        Render and validate proxy setting with given variables.
+        :param candidate: raw proxy setting as `dict`
+        :param variables: variables to render template in proxy setting.
+        :return: A `dict` contains rendered proxy setting.
+        """
+        if not candidate:
+            return {}
+
+        proxy = self._render_from_dict(candidate, variables)
+
+        enabled = proxy.get('enabled', '0')
+        if not is_valid_bool(enabled):
+            raise ValueError(
+                'Proxy "enabled" expect to be bool type: {}'.format(enabled)
+            )
+
+        proxy['enabled'] = is_true(enabled)
+
+        host, port = proxy.get('host'), proxy.get('port')
+
+        if host or port:
+            if not host:
+                raise ValueError('Proxy "host" must not be empty')
+
+            if not is_valid_port(port):
+                raise ValueError(
+                    'Proxy "port" expect to be in range [1,65535]: %s' % port
+                )
+
+        # proxy type default to 'http'
+        proxy_type = proxy.get('type')
+        proxy_type = proxy_type.lower() if proxy_type else 'http'
+        if proxy_type not in _PROXY_TYPES:
+            raise ValueError(
+                'Proxy "type" expect to be one of [{}]: {}'.format(
+                    ','.join(_PROXY_TYPES), proxy_type)
+            )
+        else:
+            proxy['type'] = proxy_type
+
+        # proxy rdns default to '0'
+        proxy_rdns = proxy.get('rdns', '0')
+        if not is_valid_bool(proxy_rdns):
+            raise ValueError(
+                'Proxy "rdns" expect to be bool type: {}'.format(proxy_rdns)
+            )
+        else:
+            proxy['rdns'] = is_true(proxy_rdns)
+
+        return proxy
+
+    @staticmethod
+    def _get_log_level(level_name):
+        if level_name:
+            level_name = level_name.upper().strip()
+
+            for k, v in _LOGGING_LEVELS.iteritems():
+                if k.startswith(level_name):
+                    return v
+
+        _logger.warning(
+            'The log level "%s" is invalid, set it to default: "%s"',
+            level_name, _DEFAULT_LOG_LEVEL
+        )
+
+        return _LOGGING_LEVELS[_DEFAULT_LOG_LEVEL]
+
+    def _load_logging(self, log_setting, variables):
+        logger = self._render_from_dict(log_setting, variables)
+
+        logger['level'] = self._get_log_level(logger.get('level'))
+
+        return logger
+
+    def _load_global_setting(self, candidate, variables):
+        """
+        Load and render global setting with variables.
+        :param candidate: Global setting as a `dict`
+        :param variables: variables from context to render setting
+        :return: A `Munch` object
+        """
+        candidate = candidate or {}
+        proxy_setting = self._load_proxy(candidate.get('proxy'), variables)
+        log_setting = self._load_logging(candidate.get('logging'), variables)
+
+        return munchify({'proxy': proxy_setting, 'logging': log_setting})
+
+    @staticmethod
+    def _load_authorization(candidate):
+        if candidate is None:
+            return None
+        auth_type = candidate['type'].lower()
+
+        if auth_type not in _AUTH_TYPES:
+            raise ValueError(
+                'Auth type expect to be one of [{}]: {}'.format(
+                    ','.join(_AUTH_TYPES.keys()), auth_type)
+            )
+        return _AUTH_TYPES[auth_type](candidate['options'])
+
+    def _load_options(self, options):
+        return Request(
+            auth=self._load_authorization(options.get('auth')),
+            url=options['url'],
+            method=options.get('method', 'GET'),
+            header=options.get('headers', {}),
+            body=options.get('body', {})
+        )
+
+    @staticmethod
+    def _validate_method(method):
+        if lookup_method(method) is None:
+            raise ValueError('Unimplemented method: {}'.format(method))
+
+    def _parse_tasks(self, raw_tasks):
+        tasks = []
+        for item in raw_tasks:
+            self._validate_method(item['method'])
+            tasks.append(Task(item['input'], item['method'], item.get('output')))
+        return tasks
+
+    def _parse_conditions(self, raw_conditions):
+        conditions = []
+        for item in raw_conditions:
+            self._validate_method(item['method'])
+            conditions.append(Condition(item['input'], item['method']))
+        return conditions
+
+    @staticmethod
+    def _load_checkpoint(checkpoint):
+        if not checkpoint:
+            return None
+        return Checkpoint(
+            checkpoint.get('namespace', []), checkpoint['content'])
+
+    def _load_iteration_mode(self, iteration_mode):
+        count = iteration_mode.get('iteration_count', '0')
+        try:
+            iteration_count = int(count)
+        except ValueError:
+            raise ValueError(
+                '"iteration_count" must be an integer: %s' % count)
+
+        stop_conditions = self._parse_conditions(
+            iteration_mode['stop_conditions'])
+
+        return IterationMode(iteration_count=iteration_count,
+                             conditions=stop_conditions)
+
+    def _load_processor(self, processor):
+        skip_conditions = self._parse_conditions(
+            processor.get('skip_conditions', [])
+        )
+        pipeline = self._parse_tasks(processor.get('pipeline', []))
+        return Processor(
+            skip_conditions=skip_conditions,
+            pipeline=pipeline
+        )
+
+    def _load_request(self, request):
+        options = self._load_options(request['request'])
+
+        pre_process = self._load_processor(request.get('pre_process', {}))
+        post_process = self._load_processor(request['post_process'])
+        checkpoint = self._load_checkpoint(request.get('checkpoint'))
+        iteration_mode = self._load_iteration_mode(request['iteration_mode'])
+
+        return munchify({
+            'request': options,
+            'pre_process': pre_process,
+            'post_process': post_process,
+            'checkpoint': checkpoint,
+            'iteration_mode': iteration_mode,
+        })
+
+    def load(self, definition, schema_file, context):
+        """Load cloud connect configuration from a `dict` and validate
+        it with schema and global settings will be rendered.
+        :param schema_file: Schema file location used to validate config.
+        :param definition: A dictionary contains raw configs.
+        :param context: variables to render template in global setting.
+        :return: A `Munch` object.
+        """
+        try:
+            validate(definition, self._get_schema_from_file(schema_file))
+        except ValidationError:
+            raise ConfigException(
+                'Failed to validate interface with schema: {}'.format(
+                    traceback.format_exc()))
+
+        try:
+            global_settings = self._load_global_setting(
+                definition.get('global_settings'), context
+            )
+
+            requests = [self._load_request(item) for item in definition['requests']]
+
+            return munchify({
+                'meta': munchify(definition['meta']),
+                'tokens': definition['tokens'],
+                'global_settings': global_settings,
+                'requests': requests,
+            })
+        except Exception as ex:
+            error = 'Unable to load configuration: %s' % str(ex)
+            _logger.exception(error)
+            raise ConfigException(error)
+
+
+_loader_and_schema_by_version = {
+    r'1\.0\.0': (CloudConnectConfigLoaderV1, 'schema_1_0_0.json'),
+}
+
+
+def get_loader_by_version(version):
+    """ Instantiate a configuration loader on basis of a given version.
+    A `ConfigException` will raised if the version is not supported.
+    :param version: Version to lookup config loader.
+    :return: A config loader.
+    """
+    for support_version in _loader_and_schema_by_version:
+        if re.match(support_version, version):
+            loader_cls, schema = _loader_and_schema_by_version[support_version]
+            return loader_cls(), schema
+
+    raise ConfigException(
+        'Unsupported schema version {}, current supported'
+        ' versions should match these regex [{}]'.format(version, ','.join(
+            _loader_and_schema_by_version))
+    )

package/bin/da_ess_contentupdate/cloudconnectlib/configuration/schema_1_0_0.json

@@ -0,0 +1,344 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "additionalProperties": false,
+  "definitions": {
+    "Authentication": {
+      "additionalProperties": false,
+      "properties": {
+        "options": {
+          "type": "object"
+        },
+        "type": {
+          "enum": [
+            "digest",
+            "basic_auth"
+          ],
+          "type": "string"
+        }
+      },
+      "required": [
+        "type"
+      ],
+      "type": "object"
+    },
+    "Checkpoint": {
+      "additionalProperties": false,
+      "properties": {
+        "content": {
+          "type": "object"
+        },
+        "namespace": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        }
+      },
+      "required": [
+        "content"
+      ],
+      "type": "object"
+    },
+    "GlobalSettings": {
+      "additionalProperties": false,
+      "properties": {
+        "logging": {
+          "properties": {
+            "level": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "proxy": {
+          "$ref": "#/definitions/Proxy"
+        }
+      },
+      "type": "object"
+    },
+    "IterationMode": {
+      "additionalProperties": false,
+      "properties": {
+        "iteration_count": {
+          "oneOf": [
+            {
+              "pattern": "^[+-]?[1-9]\\d*|0$",
+              "type": "string"
+            },
+            {
+              "type": "integer"
+            }
+          ]
+        },
+        "stop_conditions": {
+          "items": {
+            "additionalProperties": false,
+            "properties": {
+              "input": {
+                "items": {
+                  "type": "string"
+                },
+                "type": "array"
+              },
+              "method": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "input",
+              "method"
+            ],
+            "type": "object"
+          },
+          "type": "array"
+        }
+      },
+      "type": "object"
+    },
+    "Meta": {
+      "additionalProperties": false,
+      "properties": {
+        "apiVersion": {
+          "pattern": "(?:\\d{1,3}\\.){2}[\\w\\-]{1,15}",
+          "type": "string"
+        }
+      },
+      "required": [
+        "apiVersion"
+      ],
+      "type": "object"
+    },
+    "Proxy": {
+      "additionalProperties": false,
+      "properties": {
+        "enabled": {
+          "default": false,
+          "oneOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "integer"
+            }
+          ]
+        },
+        "host": {
+          "type": "string"
+        },
+        "password": {
+          "type": "string"
+        },
+        "port": {
+          "oneOf": [
+            {
+              "type": "string"
+            },
+            {
+              "exclusiveMaximum": true,
+              "exclusiveMinimum": true,
+              "maximum": 65535,
+              "minimum": 1,
+              "type": "integer"
+            }
+          ]
+        },
+        "rdns": {
+          "type": "string"
+        },
+        "type": {
+          "type": "string"
+        },
+        "username": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "host",
+        "port"
+      ],
+      "type": "object"
+    },
+    "Request": {
+      "additionalProperties": false,
+      "properties": {
+        "checkpoint": {
+          "$ref": "#/definitions/Checkpoint"
+        },
+        "iteration_mode": {
+          "$ref": "#/definitions/IterationMode"
+        },
+        "post_process": {
+          "additionalProperties": false,
+          "properties": {
+            "pipeline": {
+              "items": {
+                "additionalProperties": false,
+                "properties": {
+                  "input": {
+                    "items": {
+                      "type": "string"
+                    },
+                    "type": "array"
+                  },
+                  "method": {
+                    "type": "string"
+                  },
+                  "output": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "input",
+                  "method"
+                ],
+                "type": "object"
+              },
+              "type": "array"
+            },
+            "skip_conditions": {
+              "items": {
+                "additionalProperties": false,
+                "properties": {
+                  "input": {
+                    "items": {
+                      "type": "string"
+                    },
+                    "type": "array"
+                  },
+                  "method": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "input",
+                  "method"
+                ],
+                "type": "object"
+              },
+              "type": "array"
+            }
+          },
+          "type": "object"
+        },
+        "pre_process": {
+          "additionalProperties": false,
+          "properties": {
+            "pipeline": {
+              "items": {
+                "additionalProperties": false,
+                "properties": {
+                  "input": {
+                    "items": {
+                      "type": "string"
+                    },
+                    "type": "array"
+                  },
+                  "method": {
+                    "type": "string"
+                  },
+                  "output": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "input",
+                  "method"
+                ],
+                "type": "object"
+              },
+              "type": "array"
+            },
+            "skip_conditions": {
+              "items": {
+                "additionalProperties": false,
+                "properties": {
+                  "input": {
+                    "items": {
+                      "type": "string"
+                    },
+                    "type": "array"
+                  },
+                  "method": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "input",
+                  "method"
+                ],
+                "type": "object"
+              },
+              "type": "array"
+            }
+          },
+          "type": "object"
+        },
+        "request": {
+          "additionalProperties": false,
+          "properties": {
+            "auth": {
+              "$ref": "#/definitions/Authentication"
+            },
+            "body": {
+              "type": "object"
+            },
+            "headers": {
+              "type": "object"
+            },
+            "method": {
+              "default": "GET",
+              "enum": [
+                "GET",
+                "POST"
+              ],
+              "type": "string"
+            },
+            "url": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "url"
+          ],
+          "type": "object"
+        }
+      },
+      "required": [
+        "request",
+        "post_process",
+        "iteration_mode"
+      ],
+      "type": "object"
+    }
+  },
+  "properties": {
+    "global_settings": {
+      "$ref": "#/definitions/GlobalSettings"
+    },
+    "meta": {
+      "$ref": "#/definitions/Meta"
+    },
+    "requests": {
+      "items": {
+        "$ref": "#/definitions/Request"
+      },
+      "minItems": 1,
+      "type": "array"
+    },
+    "tokens": {
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    }
+  },
+  "required": [
+    "meta",
+    "tokens",
+    "requests"
+  ],
+  "type": "object"
+}

package/bin/da_ess_contentupdate/cloudconnectlib/core/__init__.py

@@ -0,0 +1,2 @@
+from .engine import CloudConnectEngine
+from .exceptions import ConfigException, HTTPError

package/bin/da_ess_contentupdate/cloudconnectlib/core/cacerts/ca_certs_locater.py

@@ -0,0 +1,130 @@
+"""
+`ca_certs_locater` is a lib for extending httplib2 to allow system certificate store to be used when
+verifying SSL certificates, to enable this lib, you should add it to your python import path before
+initializing httplib2. As we're not trying to implement SSL certificate RFCs, parsing and validating
+certificates are not included.
+"""
+
+import atexit
+import os
+import os.path as op
+import ssl
+import sys
+
+TEMP_CERT_FILE_NAME = 'httplib2_merged_certificates_{}.crt'
+LINUX_CERT_PATH_1 = '/etc/pki/tls/certs/ca-bundle.crt'  # RedHat
+LINUX_CERT_PATH_2 = '/etc/ssl/certs/ca-certificates.crt'  # Debian
+DARWIN_CERT_PATH = '/usr/local/etc/openssl/cert.pem'
+HTTPLIB2_CA_CERT_FILE_NAME = 'cacerts.txt'
+
+TEMP_CERT_FILE_PATH = None
+
+
+def get():
+    """
+    Returns: a path to generated certificate authority file
+
+    """
+    try:
+        return _get()
+    except (IOError, OSError, ssl.SSLError):
+        _fallback()  # IO and SSL relative errors should be swallowed to protect the HTTP request
+
+
+def _get():
+    global TEMP_CERT_FILE_PATH
+    # also check file existence as it's possible for the temp file to be deleted
+    if TEMP_CERT_FILE_PATH is None or not os.path.exists(TEMP_CERT_FILE_PATH):
+        temp_cert_file_path = _generate_temp_cert_file_name()
+        ssl_ca_certs = _read_ssl_default_ca_certs()
+        if not ssl_ca_certs:
+            # it's possible the ca load path is not well configured, try some typical paths
+            ssl_ca_certs = _read_platform_pem_cert_file()
+
+        if ssl_ca_certs:  # only update temp cert file when there's additional PEM certs found
+            cert_files = [ssl_ca_certs, _read_httplib2_default_certs()]
+            _update_temp_cert_file(temp_cert_file_path, cert_files)
+            TEMP_CERT_FILE_PATH = temp_cert_file_path
+        else:
+            _fallback()
+
+    return TEMP_CERT_FILE_PATH
+
+
+def _fallback():
+    """
+        Give up the loading process by throwing specified exception, httplib2 will then use its
+        bundled certificates
+    """
+    raise ImportError('Unable to load system certificate authority files')
+
+
+def _read_platform_pem_cert_file():
+    if sys.platform.startswith('linux'):
+        pem_files = [_read_pem_file(LINUX_CERT_PATH_1), _read_pem_file(LINUX_CERT_PATH_2)]
+        return '\n'.join(filter(None, pem_files))
+    elif sys.platform.startswith('darwin'):
+        return _read_pem_file(DARWIN_CERT_PATH)
+    else:
+        return ""
+
+
+def _read_ssl_default_ca_certs():
+    # it's not guaranteed to return PEM formatted certs when `binary_form` is False
+    der_certs = ssl.create_default_context().get_ca_certs(binary_form=True)
+    pem_certs = [ssl.DER_cert_to_PEM_cert(der_cert_bytes) for der_cert_bytes in der_certs]
+    return '\n'.join(pem_certs)
+
+
+def _read_httplib2_default_certs():
+    import httplib2  # import error should not happen here, and will be well handled by outer called
+    httplib_dir = os.path.dirname(os.path.abspath(httplib2.__file__))
+    ca_certs_path = os.path.join(httplib_dir, HTTPLIB2_CA_CERT_FILE_NAME)
+    return _read_pem_file(ca_certs_path)
+
+
+def _read_pem_file(path):
+    if os.path.exists(path):
+        with open(path, mode='r') as pem_file:
+            return pem_file.read()
+    else:
+        return ""
+
+
+def _update_temp_cert_file(temp_file, pem_texts):
+    with open(temp_file, mode='w') as temp_cert_file:
+        for pem_text in pem_texts:
+            if len(pem_text) > 0:
+                temp_cert_file.write(pem_text + '\n')
+        temp_cert_file.flush()
+    atexit.register(_do_safe_remove, temp_file)
+
+
+def _do_safe_remove(file_path):
+    if os.path.exists(file_path):
+        try:
+            os.remove(file_path)
+        except:
+            pass
+
+
+def _get_temp_cert_file_dir():
+    import __main__
+    app_root = op.dirname(op.dirname(op.abspath(__main__.__file__)))
+
+    temp_dir = op.join(app_root, 'temp_certs')
+    if not op.isdir(temp_dir):
+        try:
+            os.mkdir(temp_dir)
+        except:
+            pass
+    for candidate in ['temp_certs', 'local', 'default']:
+        dir_path = op.join(app_root, candidate)
+        if op.isdir(dir_path):
+            return dir_path
+    return app_root
+
+
+def _generate_temp_cert_file_name():
+    file_name = TEMP_CERT_FILE_NAME.format(os.getpid())
+    return os.path.join(_get_temp_cert_file_dir(), file_name)

package/bin/da_ess_contentupdate/cloudconnectlib/core/defaults.py

@@ -0,0 +1,20 @@
+"""Default config for cloud connect"""
+
+timeout = 120  # request timeout is two minutes
+
+disable_ssl_cert_validation = False  # default enable SSL validation
+
+success_statuses = (200, 201)  # statuses be treated as success.
+
+# response status which need to retry.
+retry_statuses = (429, 500, 501, 502, 503, 504, 505, 506, 507,
+                  509, 510, 511)
+
+# response status which need print a warning log.
+warning_statuses = (203, 204, 205, 206, 207, 208, 226,
+                    300, 301, 302, 303, 304, 305, 306, 307, 308)
+retries = 3  # Default maximum retry times.
+
+max_iteration_count = 100  # maximum iteration loop count
+
+charset = 'utf-8'  # Default response charset if not found in response header

package/bin/da_ess_contentupdate/cloudconnectlib/core/engine.py

@@ -0,0 +1,327 @@
+import json
+import threading
+
+from . import defaults
+from .exceptions import HTTPError, StopCCEIteration
+from .http import HTTPRequest
+from ..common.log import get_cc_logger
+
+_logger = get_cc_logger()
+
+
+class CloudConnectEngine(object):
+    """The cloud connect engine to process request instantiated
+     from user options."""
+
+    def __init__(self):
+        self._stopped = False
+        self._running_job = None
+
+    @staticmethod
+    def _set_logging(log_setting):
+        _logger.set_level(log_setting.level)
+
+    def start(self, context, config, checkpoint_mgr):
+        """Start current client instance to execute each request parsed
+         from config.
+        """
+        if not config:
+            raise ValueError('Config must not be empty')
+
+        context = context or {}
+        global_setting = config.global_settings
+
+        CloudConnectEngine._set_logging(global_setting.logging)
+
+        _logger.info('Start to execute requests jobs.')
+        processed = 0
+
+        for request in config.requests:
+            job = Job(
+                request=request,
+                context=context,
+                checkpoint_mgr=checkpoint_mgr,
+                proxy=global_setting.proxy,
+            )
+            self._running_job = job
+            job.run()
+
+            processed += 1
+            _logger.info('%s job(s) process finished', processed)
+
+            if self._stopped:
+                _logger.info(
+                    'Engine has been stopped, stopping to execute jobs.')
+                break
+
+        self._stopped = True
+        _logger.info('Engine executing finished')
+
+    def stop(self):
+        """Stops engine and running job. Do nothing if engine already
+        been stopped."""
+        if self._stopped:
+            _logger.info('Engine already stopped, do nothing.')
+            return
+
+        _logger.info('Stopping engine')
+
+        if self._running_job:
+            _logger.info('Attempting to stop the running job.')
+            self._running_job.terminate()
+            _logger.info('Stopping job finished.')
+
+        self._stopped = True
+
+
+class Job(object):
+    """Job class represents a single request to send HTTP request until
+    reached it's stop condition.
+    """
+
+    def __init__(self, request, context, checkpoint_mgr, proxy=None):
+        """
+        Constructs a `Job` with properties request, context and a
+         optional proxy setting.
+        :param request: A `Request` instance which contains request settings.
+        :param context: A values set contains initial values for template
+         variables.
+        :param proxy: A optional `Proxy` object contains proxy related
+         settings.
+        """
+        self._request = request
+        self._context = context
+        self._checkpoint_mgr = checkpoint_mgr
+        self._client = HTTPRequest(proxy)
+        self._stopped = True
+        self._should_stop = False
+
+        self._request_iterated_count = 0
+        self._iteration_mode = self._request.iteration_mode
+        self._max_iteration_count = self._get_max_iteration_count()
+
+        self._running_thread = None
+        self._terminated = threading.Event()
+
+    def _get_max_iteration_count(self):
+        mode_max_count = self._iteration_mode.iteration_count
+        default_max_count = defaults.max_iteration_count
+        return min(default_max_count, mode_max_count) \
+            if mode_max_count > 0 else default_max_count
+
+    def terminate(self, block=True, timeout=30):
+        """Terminate this job, the current thread will blocked util
+        the job is terminate finished if block is True """
+        if self.is_stopped():
+            _logger.info('Job already been stopped.')
+            return
+
+        if self._running_thread == threading.current_thread():
+            _logger.warning('Job cannot terminate itself.')
+            return
+
+        _logger.info('Stopping job')
+        self._should_stop = True
+
+        if not block:
+            return
+        if not self._terminated.wait(timeout):
+            _logger.warning('Terminating job timeout.')
+
+    def _set_context(self, key, value):
+        self._context[key] = value
+
+    def _execute_tasks(self, tasks):
+        if not tasks:
+            return
+        for task in tasks:
+            if self._check_should_stop():
+                return
+            self._context.update(task.execute(self._context))
+
+    def _on_pre_process(self):
+        """
+        Execute tasks in pre process one by one if condition satisfied.
+        """
+        pre_processor = self._request.pre_process
+
+        if pre_processor.should_skipped(self._context):
+            _logger.info('Skip pre process condition satisfied, do nothing')
+            return
+
+        tasks = pre_processor.pipeline
+        _logger.debug(
+            'Got %s tasks need be executed before process', len(tasks))
+        self._execute_tasks(tasks)
+
+    def _on_post_process(self):
+        """
+        Execute tasks in post process one by one if condition satisfied.
+        """
+        post_processor = self._request.post_process
+
+        if post_processor.should_skipped(self._context):
+            _logger.info('Skip post process condition satisfied, '
+                         'do nothing')
+            return
+
+        tasks = post_processor.pipeline
+        _logger.debug(
+            'Got %s tasks need to be executed after process', len(tasks)
+        )
+        self._execute_tasks(tasks)
+
+    def _update_checkpoint(self):
+        """Updates checkpoint based on checkpoint namespace and content."""
+        checkpoint = self._request.checkpoint
+        if not checkpoint:
+            _logger.info('Checkpoint not specified, do not update it.')
+            return
+
+        self._checkpoint_mgr.update_ckpt(
+            checkpoint.normalize_content(self._context),
+            namespaces=checkpoint.normalize_namespace(self._context),
+        )
+
+    def _get_checkpoint(self):
+        checkpoint = self._request.checkpoint
+        if not checkpoint:
+            _logger.info('Checkpoint not specified, do not read it.')
+            return
+
+        namespaces = checkpoint.normalize_namespace(self._context)
+        checkpoint = self._checkpoint_mgr.get_ckpt(namespaces)
+        if checkpoint:
+            self._context.update(checkpoint)
+
+    def _is_stoppable(self):
+        """Check if repeat mode conditions satisfied."""
+        if self._request_iterated_count >= self._max_iteration_count:
+            _logger.info(
+                'Job iteration count is %s, current request count is %s,'
+                ' stop condition satisfied.',
+                self._max_iteration_count, self._request_iterated_count
+            )
+            return True
+
+        if self._iteration_mode.passed(self._context):
+            _logger.info('Job stop condition satisfied.')
+            return True
+
+        return False
+
+    def is_stopped(self):
+        """Return if this job is stopped."""
+        return self._stopped
+
+    def run(self):
+        """Start job and exit util meet stop condition. """
+        _logger.info('Start to process job')
+
+        self._stopped = False
+        try:
+            self._running_thread = threading.current_thread()
+            self._run()
+        except Exception:
+            _logger.exception('Error encountered while running job.')
+            raise
+        finally:
+            self._terminated.set()
+            self._stopped = True
+
+        _logger.info('Job processing finished')
+
+    def _check_should_stop(self):
+        if self._should_stop:
+            _logger.info('Job should been stopped.')
+        return self._should_stop
+
+    def _run(self):
+        request = self._request.request
+        method = request.method
+        authorizer = request.auth
+        self._get_checkpoint()
+
+        while 1:
+            if self._check_should_stop():
+                return
+
+            try:
+                self._on_pre_process()
+            except StopCCEIteration:
+                _logger.info('Stop iteration command in pre process is received, exit job now.')
+                return
+
+            url = request.normalize_url(self._context)
+            header = request.normalize_header(self._context)
+            body = request.normalize_body(self._context)
+            body_json = json.dumps(body) if body else None
+
+            if authorizer:
+                authorizer(header, self._context)
+
+            if self._check_should_stop():
+                return
+
+            response, need_terminate = \
+                self._send_request(url, method, header, body=body_json)
+
+            if need_terminate:
+                _logger.info('This job need to be terminated.')
+                break
+
+            self._request_iterated_count += 1
+            self._set_context('__response__', response)
+            if self._check_should_stop():
+                return
+
+            try:
+                self._on_post_process()
+            except StopCCEIteration:
+                _logger.info('Stop iteration command in post process is received, exit job now.')
+                return
+
+            if self._check_should_stop():
+                return
+            self._update_checkpoint()
+
+            if self._is_stoppable():
+                _logger.info('Stop condition reached, exit job now')
+                break
+
+    def _send_request(self, url, method, header, body):
+        """Do send request with a simple error handling strategy. Refer to
+        https://confluence.splunk.com/display/PROD/CC+1.0+-+Detail+Design"""
+        try:
+            response = self._client.request(
+                url, method, headers=header, body=body
+            )
+        except HTTPError as error:
+            _logger.exception(
+                'HTTPError reason=%s when sending request to '
+                'url=%s method=%s', error.reason, url, method)
+            return None, True
+
+        status = response.status_code
+
+        if status in defaults.success_statuses:
+            if not (response.body or '').strip():
+                _logger.info(
+                    'The response body of request which url=%s and'
+                    ' method=%s is empty, status=%s.',
+                    url, method, status
+                )
+                return None, True
+            return response, False
+
+        error_log = ('The response status=%s for request which url=%s and'
+                     ' method=%s.') % (
+                        status, url, method
+                    )
+
+        if status in defaults.warning_statuses:
+            _logger.warning(error_log)
+        else:
+            _logger.error(error_log)
+
+        return None, True

package/bin/da_ess_contentupdate/cloudconnectlib/core/exceptions.py

@@ -0,0 +1,27 @@
+"""APP Cloud Connect errors"""
+
+
+class ConfigException(Exception):
+    """Config exception"""
+    pass
+
+
+class FuncException(Exception):
+    """Ext function call exception"""
+    pass
+
+
+class HTTPError(Exception):
+    """ HTTPError raised when HTTP request returned a error."""
+
+    def __init__(self, reason=None):
+        """
+        Initialize HTTPError with `response` object and `status`.
+        """
+        self.reason = reason
+        super(HTTPError, self).__init__(reason)
+
+
+class StopCCEIteration(Exception):
+    """Exception to exit from the engine iteration."""
+    pass

package/bin/da_ess_contentupdate/cloudconnectlib/core/ext.py

@@ -0,0 +1,336 @@
+import calendar
+import json
+import re
+import traceback
+from datetime import datetime
+
+from jsonpath_rw import parse
+from .exceptions import FuncException, StopCCEIteration
+from .pipemgr import PipeManager
+from ..common import util, log
+
+_logger = log.get_cc_logger()
+
+
+def regex_match(pattern, source, flags=0):
+    """
+    Determine whether a string is match a regex pattern.
+
+    :param pattern: regex pattern
+    :param source: candidate to match regex
+    :param flags: flags for regex match
+    :return: `True` if candidate match pattern else `False`
+    """
+    try:
+        return re.match(pattern, source, flags) is not None
+    except Exception:
+        _logger.warning(
+            'Unable to match source with pattern=%s, cause=%s',
+            pattern,
+            traceback.format_exc()
+        )
+    return False
+
+
+def regex_not_match(pattern, source, flags=0):
+    """
+    Determine whether a string is not match a regex pattern.
+
+    :param pattern: regex expression
+    :param source: candidate to match regex
+    :param flags: flags for regex match
+    :return: `True` if candidate not match pattern else `False`
+    """
+    return not regex_match(pattern, source, flags)
+
+
+def json_path(source, json_path_expr):
+    """ Extract value from string with JSONPATH expression.
+    :param json_path_expr: JSONPATH expression
+    :param source: string to extract value
+    :return: A `list` contains all values extracted
+    """
+    if not source:
+        _logger.debug('source to apply JSONPATH is empty, return empty.')
+        return ''
+
+    if isinstance(source, basestring):
+        _logger.debug(
+            'source expected is a JSON, not %s. Attempt to'
+            ' convert it to JSON',
+            type(source)
+        )
+        try:
+            source = json.loads(source)
+        except Exception as ex:
+            _logger.warning(
+                'Unable to load JSON from source: %s. '
+                'Attempt to apply JSONPATH "%s" on source directly.',
+                ex.message,
+                json_path_expr
+            )
+
+    try:
+        expression = parse(json_path_expr)
+        results = [match.value for match in expression.find(source)]
+
+        _logger.debug(
+            'Got %s elements extracted with JSONPATH expression "%s"',
+            len(results), json_path_expr
+        )
+
+        if not results:
+            return ''
+
+        return results[0] or '' if len(results) == 1 else results
+    except Exception as ex:
+        _logger.warning(
+            'Unable to apply JSONPATH expression "%s" on source,'
+            ' message=%s cause=%s',
+            json_path_expr,
+            ex.message,
+            traceback.format_exc()
+        )
+    return ''
+
+
+def splunk_xml(candidates,
+               time=None,
+               index=None,
+               host=None,
+               source=None,
+               sourcetype=None):
+    """ Wrap a event with splunk xml format.
+    :param candidates: data used to wrap as event
+    :param time: timestamp which must be empty or a valid float
+    :param index: index name for event
+    :param host: host for event
+    :param source: source for event
+    :param sourcetype: sourcetype for event
+    :return: A wrapped event with splunk xml format.
+    """
+    if not isinstance(candidates, (list, tuple)):
+        candidates = [candidates]
+
+    time = time or None
+    if time:
+        try:
+            time = float(time)
+        except ValueError:
+            _logger.warning(
+                '"time" %s is expected to be a float, set "time" to None',
+                time
+            )
+            time = None
+
+    return util.format_events(
+        candidates,
+        time=time,
+        index=index,
+        host=host,
+        source=source,
+        sourcetype=sourcetype
+    )
+
+
+def std_output(candidates):
+    """ Output a string to stdout.
+    :param candidates: List of string to output to stdout or a single string.
+    """
+    if isinstance(candidates, basestring):
+        candidates = [candidates]
+
+    all_str = True
+    for candidate in candidates:
+        if all_str and not isinstance(candidate, basestring):
+            all_str = False
+            _logger.warning(
+                'The type of data needs to print is "%s" rather than'
+                ' basestring',
+                type(candidate)
+            )
+
+        if not PipeManager().write_events(candidate):
+            raise FuncException('Fail to output data to stdout. The event'
+                                ' writer is stopped or encountered exception')
+
+    _logger.debug('Writing events to stdout finished.')
+    return True
+
+
+def _parse_json(source, json_path_expr=None):
+    if not source:
+        _logger.debug('Unable to parse JSON from empty source, return empty.')
+        return {}
+
+    if json_path_expr:
+        _logger.debug(
+            'Try to extract JSON from source with JSONPATH expression: %s, ',
+            json_path_expr
+        )
+        source = json_path(source, json_path_expr)
+
+    elif isinstance(source, basestring):
+        source = json.loads(source)
+
+    return source
+
+
+def json_empty(source, json_path_expr=None):
+    """Check whether a JSON is empty, return True only if the JSON to
+     check is a valid JSON and is empty.
+    :param json_path_expr: A optional JSONPATH expression
+    :param source: source to extract JSON
+    :return: `True` if the result JSON is empty
+    """
+    try:
+        data = _parse_json(source, json_path_expr)
+
+        if isinstance(data, (list, tuple)):
+            return all(len(ele) == 0 for ele in data)
+        return len(data) == 0
+    except Exception as ex:
+        _logger.warning(
+            'Unable to determine whether source is json_empty, treat it as '
+            'not json_empty: %s', ex.message
+        )
+        return False
+
+
+def json_not_empty(source, json_path_expr=None):
+    """Check if a JSON object is not empty, return True only if the
+     source is a valid JSON object and the value leading by
+     json_path_expr is empty.
+    :param json_path_expr: A optional JSONPATH expression
+    :param source: source to extract JSON
+    :return: `True` if the result JSON is not empty
+    """
+    try:
+        data = _parse_json(source, json_path_expr)
+
+        if isinstance(data, (list, tuple)):
+            return any(len(ele) > 0 for ele in data)
+        return len(data) > 0
+    except Exception as ex:
+        _logger.warning(
+            'Unable to determine whether source is json_not_empty, '
+            'treat it as not json_not_empty: %s',
+            ex.message
+        )
+        return False
+
+
+def set_var(value):
+    """Set a variable which name should be specified in `output` with value"""
+    return value
+
+
+def _fix_microsecond_format(fmt, micros):
+    """
+    implement %Nf so that user can control the digital number of microsecond.
+    If number of % is even, don't do replacement.
+    If N is not in [1-6], don't do replacement.
+    If time length m is less than N, convert it to 6 digitals and return N
+    digitals.
+    """
+    micros = str(micros).zfill(6)
+
+    def do_replacement(x, micros):
+        if int(x.group(1)) in range(1, 7) and len(x.group()) % 2:
+            return x.group().replace('%' + x.group(1) + 'f',
+                                     micros[:min(int(x.group(1)), len(micros))])
+        return x.group()
+
+    return re.sub(r'%+([1-6])f', lambda x: do_replacement(x, micros), fmt)
+
+
+def _fix_timestamp_format(fmt, timestamp):
+    """Replace '%s' in time format with timestamp if the number
+        of '%' before 's' is odd."""
+    return re.sub(
+        r'%+s',
+        (
+            lambda x:
+            x.group() if len(x.group()) % 2 else x.group().replace('%s',
+                                                                   timestamp)
+        ),
+        fmt
+    )
+
+
+def time_str2str(date_string, from_format, to_format):
+    """Convert a date string with given format to another format. Return
+    the original date string if it's type is not string or failed to parse or
+    convert it with format."""
+    if not isinstance(date_string, basestring):
+        _logger.warning(
+            '"date_string" must be a string type, found %s,'
+            ' return the original date_string directly.',
+            type(date_string)
+        )
+        return date_string
+
+    try:
+        dt = datetime.strptime(date_string, from_format)
+        # Need to pre process '%s' in to_format here because '%s' is not
+        # available on all platforms. Even on supported platforms, the
+        # result may be different because it depends on implementation on each
+        # platform. Replace it with UTC timestamp here directly.
+        if to_format:
+            timestamp = calendar.timegm(dt.timetuple())
+            to_format = _fix_timestamp_format(to_format, str(timestamp))
+            to_format = _fix_microsecond_format(to_format, str(dt.microsecond))
+        return dt.strftime(to_format)
+    except Exception:
+        _logger.warning(
+            'Unable to convert date_string "%s" from format "%s" to "%s",'
+            ' return the original date_string, cause=%s',
+            date_string,
+            from_format,
+            to_format,
+            traceback.format_exc()
+        )
+    return date_string
+
+
+def is_true(value):
+    """Determine whether value is True"""
+    return str(value).strip().lower() == 'true'
+
+
+def exit_if_true(value):
+    """Raise a StopCCEIteration exception if value is True"""
+    if is_true(value):
+        raise StopCCEIteration
+
+
+def assert_true(value, message=None):
+    """Assert value is True"""
+    if not is_true(value):
+        raise AssertionError(
+            message or '"{value}" is not true'.format(value=value)
+        )
+
+
+_extension_functions = {
+    'assert_true': assert_true,
+    'exit_if_true': exit_if_true,
+    'is_true': is_true,
+    'regex_match': regex_match,
+    'regex_not_match': regex_not_match,
+    'set_var': set_var,
+    'splunk_xml': splunk_xml,
+    'std_output': std_output,
+    'json_path': json_path,
+    'json_empty': json_empty,
+    'json_not_empty': json_not_empty,
+    'time_str2str': time_str2str,
+}
+
+
+def lookup_method(name):
+    """ Find a predefined function with given function name.
+    :param name: function name.
+    :return: A function with given name.
+    """
+    return _extension_functions.get(name)

package/bin/da_ess_contentupdate/cloudconnectlib/core/http.py

@@ -0,0 +1,234 @@
+import time
+import traceback
+
+from httplib2 import ProxyInfo, Http, socks, SSLHandshakeError
+from solnlib.packages.requests import PreparedRequest, utils
+from . import defaults
+from .exceptions import HTTPError
+from ..common.log import get_cc_logger
+
+_logger = get_cc_logger()
+
+
+class HTTPResponse(object):
+    """
+    HTTPResponse class wraps response of HTTP request for later use.
+    """
+
+    def __init__(self, response, content):
+        """Construct a HTTPResponse from response and content returned
+        with httplib2 request"""
+        self._status_code = response.status
+        self._header = response
+        self._body = self._decode_content(response, content)
+
+    @staticmethod
+    def _decode_content(response, content):
+        if not content:
+            return ''
+
+        charset = utils.get_encoding_from_headers(response)
+
+        if charset is None:
+            charset = defaults.charset
+            _logger.info(
+                'Unable to find charset in response headers,'
+                ' set it to default "%s"', charset
+            )
+
+        _logger.info('Decoding response content with charset=%s', charset)
+
+        try:
+            return content.decode(charset, errors='replace')
+        except Exception as ex:
+            _logger.warning(
+                'Failure decoding response content with charset=%s,'
+                ' decode it with utf-8: %s',
+                charset, ex.message
+            )
+
+        return content.decode('utf-8', errors='replace')
+
+    @property
+    def header(self):
+        return self._header
+
+    @property
+    def body(self):
+        """
+        Return response body as a `string`.
+        :return: A `string`
+        """
+        return self._body
+
+    @property
+    def status_code(self):
+        """
+        Return response status code.
+        :return: A `integer`
+        """
+        return self._status_code
+
+
+def _make_prepare_url_func():
+    """Expose prepare_url in `PreparedRequest`"""
+    pr = PreparedRequest()
+
+    def prepare_url(url, params=None):
+        """Prepare the given HTTP URL with ability provided in requests lib.
+        For some illegal characters in URL or parameters like space(' ') will
+        be escaped to make sure we can request the correct URL."""
+        pr.prepare_url(url, params=params)
+        return pr.url
+
+    return prepare_url
+
+
+class HTTPRequest(object):
+    """
+    HTTPRequest class represents a single request to send HTTP request until
+    reached it's stop condition.
+    """
+
+    _PROXY_TYPE = {
+        'http': socks.PROXY_TYPE_HTTP,
+        'http_no_tunnel': socks.PROXY_TYPE_HTTP_NO_TUNNEL,
+        'socks4': socks.PROXY_TYPE_SOCKS4,
+        'socks5': socks.PROXY_TYPE_SOCKS5,
+    }
+
+    def __init__(self, proxy=None):
+        """Constructs a `HTTPRequest` with a optional proxy setting.
+        :param proxy: A optional `Proxy` object contains proxy related
+         settings.
+        """
+        self._proxy_info = self._prepare_proxy_info(proxy)
+        self._connection = None
+        self._prepare_url_func = _make_prepare_url_func()
+
+    def _send_request(self, uri, method, headers=None, body=None):
+        """Do send request to target URL and validate SSL cert by default.
+        If validation failed, disable it and try again."""
+        if self._connection is None:
+            self._connection = self._build_http_connection(
+                proxy_info=self._proxy_info,
+                disable_ssl_cert_validation=False)
+
+        try:
+            return self._connection.request(
+                uri, body=body, method=method, headers=headers
+            )
+        except SSLHandshakeError:
+            _logger.warning(
+                "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verification failed. "
+                "The certificate of the https server [%s] is not trusted, "
+                "this add-on will proceed to connect with this certificate. "
+                "You may need to check the certificate and "
+                "refer to the documentation and add it to the trust list. %s",
+                uri,
+                traceback.format_exc()
+            )
+
+            self._connection = self._build_http_connection(
+                proxy_info=self._proxy_info,
+                disable_ssl_cert_validation=True
+            )
+            return self._connection.request(
+                uri, body=body, method=method, headers=headers
+            )
+
+    def request(self, url, method='GET', headers=None, body=None):
+        """
+        Invoke a request with httplib2 and return it's response.
+        :param url: url address to send request to.
+        :param method: request method `GET` by default.
+        :param headers: request headers.
+        :param body: request body.
+        :return: A `HTTPResponse` object.
+        """
+
+        if body and not isinstance(body, str):
+            raise TypeError('Request body type must be str')
+
+        if self._connection is None:
+            self._connection = self._build_http_connection(self._proxy_info)
+
+        try:
+            uri = self._prepare_url_func(url)
+        except Exception:
+            _logger.warning(
+                'Failed to encode url=%s: %s, use original url directly',
+                url, traceback.format_exc()
+            )
+            uri = url
+
+        _logger.info('Preparing to invoke request to [%s]', uri)
+
+        result = self._do_request(uri, method, headers, body)
+
+        _logger.info('Invoking request to [%s] finished', uri)
+
+        return result
+
+    def _prepare_proxy_info(self, proxy):
+        if not proxy or not proxy.enabled:
+            _logger.debug('Proxy is not enabled')
+            return None
+
+        username = proxy.username \
+            if 'username' in proxy and proxy.username else None
+        password = proxy.password \
+            if 'password' in proxy and proxy.password else None
+
+        proxy_type = self._PROXY_TYPE.get(proxy.type) or self._PROXY_TYPE['http']
+
+        return ProxyInfo(proxy_host=proxy.host,
+                         proxy_port=int(proxy.port),
+                         proxy_type=proxy_type,
+                         proxy_user=username,
+                         proxy_pass=password,
+                         proxy_rdns=proxy.rdns)
+
+    @staticmethod
+    def _build_http_connection(
+            proxy_info=None,
+            timeout=defaults.timeout,
+            disable_ssl_cert_validation=defaults.disable_ssl_cert_validation):
+        return Http(
+            proxy_info=proxy_info,
+            timeout=timeout,
+            disable_ssl_certificate_validation=disable_ssl_cert_validation)
+
+    @staticmethod
+    def _is_need_retry(status, retried, maximum_retries):
+        return retried < maximum_retries \
+               and status in defaults.retry_statuses
+
+    def _do_request(self, uri, method='GET', headers=None, body=None):
+        """Invokes request and auto retry with an exponential backoff
+        if the response status is configured in defaults.retry_statuses."""
+        retries = max(defaults.retries, 0)
+
+        for i in xrange(retries + 1):
+            try:
+                response, content = self._send_request(
+                    uri, body=body, method=method, headers=headers
+                )
+            except Exception as err:
+                _logger.exception(
+                    'Could not send request url=%s method=%s', uri, method)
+                raise HTTPError('HTTP Error %s' % str(err))
+
+            status = response.status
+
+            if self._is_need_retry(status, i, retries):
+                delay = 2 ** i
+                _logger.warning(
+                    'The response status=%s of request which url=%s and'
+                    ' method=%s. Retry after %s seconds.',
+                    status, uri, method, delay,
+                )
+                time.sleep(delay)
+                continue
+
+            return HTTPResponse(response, content)

package/bin/da_ess_contentupdate/cloudconnectlib/core/models.py

@@ -0,0 +1,277 @@
+import base64
+import traceback
+
+from .ext import lookup_method
+from .template import compile_template
+from ..common.log import get_cc_logger
+
+_logger = get_cc_logger()
+
+
+class _Token(object):
+    """Token class wraps a template expression"""
+
+    def __init__(self, source):
+        """Constructs _Token from source. A rendered template
+        will be created if source is string type because Jinja
+        template must be a string."""
+        self._source = source
+        self._value_for = compile_template(source) \
+            if isinstance(source, basestring) else None
+
+    def render(self, variables):
+        """Render value with variables if source is a string.
+        Otherwise return source directly."""
+        if self._value_for is None:
+            return self._source
+        try:
+            return self._value_for(variables)
+        except Exception as ex:
+            _logger.warning(
+                'Unable to render template "%s". Please make sure template is'
+                ' a valid Jinja2 template and token is exist in variables. '
+                'message=%s cause=%s',
+                self._source,
+                ex.message,
+                traceback.format_exc()
+            )
+        return self._source
+
+
+class DictToken(object):
+    """DictToken wraps a dict which value is template expression"""
+
+    def __init__(self, template_expr):
+        self._tokens = {k: _Token(v)
+                        for k, v in (template_expr or {}).iteritems()}
+
+    def render(self, variables):
+        return {k: v.render(variables) for k, v in self._tokens.iteritems()}
+
+
+class BaseAuth(object):
+    """A base class for all authorization classes"""
+
+    def __call__(self, headers, context):
+        raise NotImplementedError('Auth must be callable.')
+
+
+class BasicAuthorization(BaseAuth):
+    """BasicAuthorization class implements basic auth"""
+
+    def __init__(self, options):
+        if not options:
+            raise ValueError('Options for basic auth unexpected to be empty')
+
+        username = options.get('username')
+        if not username:
+            raise ValueError('Username is mandatory for basic auth')
+        password = options.get('password')
+        if not password:
+            raise ValueError('Password is mandatory for basic auth')
+
+        self._username = _Token(username)
+        self._password = _Token(password)
+
+    def __call__(self, headers, context):
+        username = self._username.render(context)
+        password = self._password.render(context)
+        headers['Authorization'] = 'Basic %s' % base64.encodestring(
+            username + ':' + password
+        ).strip()
+
+
+class Request(object):
+    def __init__(self, url, method, header=None, auth=None, body=None):
+        self._header = DictToken(header)
+        self._url = _Token(url)
+        self._method = method.upper()
+        self._auth = auth
+        self._body = DictToken(body)
+
+    @property
+    def header(self):
+        return self._header
+
+    @property
+    def url(self):
+        return self._url
+
+    @property
+    def method(self):
+        return self._method
+
+    @property
+    def auth(self):
+        return self._auth
+
+    @property
+    def body(self):
+        return self._body
+
+    def normalize_url(self, context):
+        """Normalize url"""
+        return self._url.render(context)
+
+    def normalize_header(self, context):
+        """Normalize headers which must be a dict which keys and values are
+        string."""
+        header = self.header.render(context)
+        return {k: str(v) for k, v in header.iteritems()}
+
+    def normalize_body(self, context):
+        """Normalize body"""
+        return self.body.render(context)
+
+
+class _Function(object):
+    def __init__(self, inputs, function):
+        self._inputs = tuple(_Token(expr) for expr in inputs or [])
+        self._function = function
+
+    @property
+    def inputs(self):
+        return self._inputs
+
+    def inputs_values(self, context):
+        """
+        Get rendered input values.
+        """
+        for arg in self._inputs:
+            yield arg.render(context)
+
+    @property
+    def function(self):
+        return self._function
+
+
+class Task(_Function):
+    """Task class wraps a task in processor pipeline"""
+
+    def __init__(self, inputs, function, output=None):
+        super(Task, self).__init__(inputs, function)
+        self._output = output
+
+    @property
+    def output(self):
+        return self._output
+
+    def execute(self, context):
+        """Execute task with arguments which rendered from context """
+        args = [arg for arg in self.inputs_values(context)]
+        caller = lookup_method(self.function)
+        output = self._output
+
+        _logger.info(
+            'Executing task method: [%s], input size: [%s], output: [%s]',
+            self.function, len(args), output
+        )
+
+        if output is None:
+            caller(*args)
+            return {}
+
+        return {output: caller(*args)}
+
+
+class Condition(_Function):
+    """A condition return the value calculated from input and function"""
+
+    def calculate(self, context):
+        """Calculate condition with input arguments rendered from context
+        and method which expected return a bool result.
+        :param context: context contains key value pairs
+        :return A bool value returned from the corresponding method
+        """
+        args = [arg for arg in self.inputs_values(context)]
+        callable_method = lookup_method(self.function)
+
+        _logger.debug(
+            'Calculating condition with method: [%s], input size: [%s]',
+            self.function, len(args)
+        )
+
+        result = callable_method(*args)
+
+        _logger.debug("Calculated result: %s", result)
+
+        return result
+
+
+class _Conditional(object):
+    """A base class for all conditional action"""
+
+    def __init__(self, conditions):
+        self._conditions = conditions or []
+
+    @property
+    def conditions(self):
+        return self._conditions
+
+    def passed(self, context):
+        """Determine if any condition is satisfied.
+        :param context: variables to render template
+        :return: `True` if all passed else `False`
+        """
+        return any(
+            condition.calculate(context) for condition in self._conditions
+        )
+
+
+class Processor(_Conditional):
+    """Processor class contains a conditional data process pipeline"""
+
+    def __init__(self, skip_conditions, pipeline):
+        super(Processor, self).__init__(skip_conditions)
+        self._pipeline = pipeline or []
+
+    @property
+    def pipeline(self):
+        return self._pipeline
+
+    def should_skipped(self, context):
+        """Determine processor if should skip process"""
+        return self.passed(context)
+
+
+class IterationMode(_Conditional):
+    def __init__(self, iteration_count, conditions):
+        super(IterationMode, self).__init__(conditions)
+        self._iteration_count = iteration_count
+
+    @property
+    def iteration_count(self):
+        return self._iteration_count
+
+    @property
+    def conditions(self):
+        return self._conditions
+
+
+class Checkpoint(object):
+    """A checkpoint includes a namespace to determine the checkpoint location
+    and a content defined the format of content stored in checkpoint."""
+
+    def __init__(self, namespace, content):
+        """Constructs checkpoint with given namespace and content template. """
+        if not content:
+            raise ValueError('Checkpoint content must not be empty')
+
+        self._namespace = tuple(_Token(expr) for expr in namespace or ())
+        self._content = DictToken(content)
+
+    @property
+    def namespace(self):
+        return self._namespace
+
+    def normalize_namespace(self, ctx):
+        """Normalize namespace with context used to render template."""
+        return [token.render(ctx) for token in self._namespace]
+
+    @property
+    def content(self):
+        return self._content
+
+    def normalize_content(self, ctx):
+        """Normalize checkpoint with context used to render template."""
+        return self._content.render(ctx)

package/bin/da_ess_contentupdate/cloudconnectlib/core/pipemgr.py

@@ -0,0 +1,14 @@
+from solnlib.pattern import Singleton
+
+
+class PipeManager(object):
+    __metaclass__ = Singleton
+
+    def __init__(self, event_writer=None):
+        self._event_writer = event_writer
+
+    def write_events(self, events):
+        if not self._event_writer:
+            print events
+            return True
+        return self._event_writer.write_events(events)

package/bin/da_ess_contentupdate/cloudconnectlib/core/template.py

@@ -0,0 +1,20 @@
+from jinja2 import Template
+import re
+
+# This pattern matches the template with only one token inside like "{{
+# token1}}", "{{ token2 }"
+PATTERN = re.compile(r"^\{\{\s*(\w+)\s*\}\}$")
+
+
+def compile_template(template):
+    _origin_template = template
+    _template = Template(template)
+
+    def translate_internal(context):
+        match = re.match(PATTERN, _origin_template)
+        if match:
+            context_var = context.get(match.groups()[0])
+            return context_var if context_var else ''
+        return _template.render(context)
+
+    return translate_internal

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/cloud_connect_mod_input.py

@@ -0,0 +1,71 @@
+import ConfigParser
+import os.path as op
+
+from .data_collection import ta_mod_input as ta_input
+from .ta_cloud_connect_client import TACloudConnectClient as CollectorCls
+from ..common.lib_util import (
+    get_main_file, get_app_root_dir, get_mod_input_script_name
+)
+
+
+def _load_options_from_inputs_spec(app_root, stanza_name):
+    input_spec_file = 'inputs.conf.spec'
+    file_path = op.join(app_root, 'README', input_spec_file)
+
+    if not op.isfile(file_path):
+        raise RuntimeError("README/%s doesn't exist" % input_spec_file)
+
+    parser = ConfigParser.RawConfigParser(allow_no_value=True)
+    parser.read(file_path)
+    options = parser.defaults().keys()
+    stanza_prefix = '%s://' % stanza_name
+
+    stanza_exist = False
+    for section in parser.sections():
+        if section == stanza_name or section.startswith(stanza_prefix):
+            options.extend(parser.options(section))
+            stanza_exist = True
+    if not stanza_exist:
+        raise RuntimeError("Stanza %s doesn't exist" % stanza_name)
+    return set(options)
+
+
+def _find_ucc_global_config_json(app_root, ucc_config_filename):
+    """Find UCC config file from all possible directories"""
+    candidates = ['local', 'default', 'bin',
+                  op.join('appserver', 'static', 'js', 'build')]
+
+    for candidate in candidates:
+        file_path = op.join(app_root, candidate, ucc_config_filename)
+        if op.isfile(file_path):
+            return file_path
+    raise RuntimeError(
+        'Unable to load %s from [%s]'
+        % (ucc_config_filename, ','.join(candidates))
+    )
+
+
+def _get_cloud_connect_config_json(script_name):
+    config_file_name = '.'.join([script_name, 'cc.json'])
+    return op.join(op.dirname(get_main_file()), config_file_name)
+
+
+def run(single_instance=False):
+    script_name = get_mod_input_script_name()
+
+    cce_config_file = _get_cloud_connect_config_json(script_name)
+
+    app_root = get_app_root_dir()
+    ucc_config_path = _find_ucc_global_config_json(
+        app_root, 'globalConfig.json'
+    )
+
+    schema_params = _load_options_from_inputs_spec(app_root, script_name)
+    ta_input.main(
+        CollectorCls,
+        schema_file_path=ucc_config_path,
+        log_suffix=script_name,
+        cc_json_file=cce_config_file,
+        schema_para_list=schema_params,
+        single_instance=single_instance
+    )

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/common/__init__.py

@@ -0,0 +1,52 @@
+import json
+import hashlib
+
+
+def load_schema_file(schema_file):
+    """
+    Load schema file.
+    """
+
+    with open(schema_file) as f:
+        ret = json.load(f)
+
+    common = ret.get("_common_", dict())
+    if common:
+        for k, v in ret.items():
+            if k == "_common_" or not isinstance(v, dict):
+                continue
+            # merge common into other values
+            for _k, _v in common.items():
+                if _k not in v:
+                    v[_k] = _v
+            ret[k] = v
+
+    return ret
+
+
+def md5_of_dict(data):
+    """
+    MD5 of dict data.
+    """
+
+    md5 = hashlib.sha256()
+    if isinstance(data, dict):
+        for key in sorted(data.keys()):
+            md5.update(repr(key))
+            md5.update(md5_of_dict(data[key]))
+    elif isinstance(data, list):
+        for item in sorted(data):
+            md5.update(md5_of_dict(item))
+    else:
+        md5.update(repr(data))
+
+    return md5.hexdigest()
+
+
+class UCCException(Exception):
+    """
+    Dispatch engine exception.
+    """
+
+    pass
+

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/common/log.py

@@ -0,0 +1,49 @@
+import logging
+from ...splunktalib.common import log as stclog
+
+
+def set_log_level(log_level):
+    """
+    Set log level.
+    """
+
+    if isinstance(log_level, basestring):
+        if log_level.upper() == "DEBUG":
+            stclog.Logs().set_level(logging.DEBUG)
+        elif log_level.upper() == "INFO":
+            stclog.Logs().set_level(logging.INFO)
+        elif log_level.upper() == "WARN":
+            stclog.Logs().set_level(logging.WARN)
+        elif log_level.upper() == "ERROR":
+            stclog.Logs().set_level(logging.ERROR)
+        elif log_level.upper() == "WARNING":
+            stclog.Logs().set_level(logging.WARNING)
+        elif log_level.upper() == "CRITICAL":
+            stclog.Logs().set_level(logging.CRITICAL)
+        else:
+            stclog.Logs().set_level(logging.INFO)
+    elif isinstance(log_level, int):
+        if log_level in [logging.DEBUG, logging.INFO, logging.ERROR,
+                         logging.WARN, logging.WARNING, logging.CRITICAL]:
+            stclog.Logs().set_level(log_level)
+        else:
+            stclog.Logs().set_level(logging.INFO)
+    else:
+        stclog.Logs().set_level(logging.INFO)
+
+
+# Global logger
+logger = stclog.Logs().get_logger("cloud_connect_engine")
+
+
+def reset_logger(name):
+    """
+    Reset logger.
+    """
+
+    stclog.reset_logger(name)
+
+    global logger
+    logger = stclog.Logs().get_logger(name)
+
+

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/common/rwlock.py

@@ -0,0 +1,71 @@
+"""
+This module provides Read-Write lock.
+"""
+
+import threading
+
+
+class _ReadLocker(object):
+    def __init__(self, lock):
+        self.lock = lock
+
+    def __enter__(self):
+        self.lock.acquire_read()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.lock.release_read()
+        return False
+
+
+class _WriteLocker(object):
+    def __init__(self, lock):
+        self.lock = lock
+
+    def __enter__(self):
+        self.lock.acquire_write()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.lock.release_write()
+        return False
+
+
+class RWLock(object):
+    """ Simple Read-Write lock.
+
+    Allow multiple read but only one writing concurrently.
+    """
+
+    def __init__(self):
+        self._condition = threading.Condition(threading.Lock())
+        self._readers = 0
+
+    def acquire_read(self):
+        self._condition.acquire()
+        self._readers += 1
+        self._condition.release()
+
+    def release_read(self):
+        self._condition.acquire()
+        try:
+            self._readers -= 1
+            if not self._readers:
+                self._condition.notifyAll()
+        finally:
+            self._condition.release()
+
+    def acquire_write(self):
+        self._condition.acquire()
+        while self._readers > 0:
+            self._condition.wait()
+
+    def release_write(self):
+        self._condition.release()
+
+    @property
+    def reader_lock(self):
+        return _ReadLocker(self)
+
+    @property
+    def writer_lock(self):
+        return _WriteLocker(self)
+

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/common/schema_meta.py

@@ -0,0 +1,7 @@
+
+FIELD_PRODUCT = '_product'
+FIELD_REST_NAMESPACE = '_rest_namespace'
+FIELD_REST_PREFIX = '_rest_prefix'
+FIELD_PROTOCOL_VERSION = '_protocol_version'
+FIELD_VERSION = '_version'
+FIELD_ENCRYPTION_FORMATTER = '_encryption_formatter'

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/config.py

@@ -0,0 +1,367 @@
+"""UCC Config Module
+This is for load/save configuration in UCC server or TA.
+The load/save action is based on specified schema.
+"""
+
+from __future__ import absolute_import
+
+import json
+import logging
+import traceback
+import time
+
+from ..splunktalib.rest import splunkd_request, code_to_msg
+from ..splunktalib.common import util as sc_util
+
+from .common import log as stulog
+from .common import UCCException
+from urllib import quote
+
+LOGGING_STOPPED = False
+
+
+def stop_logging():
+    """
+    Stop Config Logging. This is for not showing REST request error
+    while splunkd shutting down.
+    :return:
+    """
+    global LOGGING_STOPPED
+    LOGGING_STOPPED = True
+
+
+def log(msg, msgx='', level=logging.INFO, need_tb=False):
+    """
+    Logging in UCC Config Module.
+    :param msg: message content
+    :param msgx: detail info.
+    :param level: logging level
+    :param need_tb: if need logging traceback
+    :return:
+    """
+    global LOGGING_STOPPED
+    if LOGGING_STOPPED:
+        return
+
+    msgx = ' - ' + msgx if msgx else ''
+    content = 'UCC Config Module: %s%s' % (msg, msgx)
+    if need_tb:
+        stack = ''.join(traceback.format_stack())
+        content = '%s\r\n%s' % (content, stack)
+    stulog.logger.log(level, content, exc_info=1)
+
+
+class ConfigException(UCCException):
+    """Exception for UCC Config Exception
+    """
+    pass
+
+
+class Config(object):
+    """UCC Config Module
+    """
+
+    # Placeholder stands for any field
+    FIELD_PLACEHOLDER = '*'
+
+    # Head of non-processing endpoint
+    NON_PROC_ENDPOINT = '#'
+
+    # Some meta fields in UCC Config schema
+    META_FIELDS = ('_product', '_rest_namespace', '_rest_prefix',
+                   '_protocol_version', '_version',
+                   '_encryption_formatter')
+
+    # Default Values for Meta fields
+    META_FIELDS_DEFAULT = {
+        '_encryption_formatter': '',
+    }
+
+    def __init__(self, splunkd_uri, session_key, schema,
+                 user='nobody', app='-'):
+        """
+        :param splunkd_uri: the root uri of Splunk server,
+            like https://127.0.0.1:8089
+        :param session_key: session key for Splunk server
+        :param schema:
+        :param user: owner of the resources requested
+        :param app: namespace of the resources requested
+        :return:
+        """
+        self.splunkd_uri = splunkd_uri.strip('/')
+        self.session_key = session_key
+        self.user, self.app = user, app
+        self._parse_schema(schema)
+        self._check_protocol_version()
+
+    def load(self):
+        """Load Configurations in UCC according to the schema
+        It will raise exception if failing to load any endpoint,
+        because it make no sense with not complete configuration info.
+        """
+        log('"load" method in', level=logging.DEBUG)
+
+        ret = {meta_field: getattr(self, meta_field)
+               for meta_field in Config.META_FIELDS}
+
+        for ep_id, ep in self._endpoints.iteritems():
+            data = {'output_mode': 'json', '--cred--': '1'}
+
+            retries = 4
+            waiting_time = [1, 2, 2]
+            for retry in xrange(retries):
+                resp, cont = splunkd_request(
+                    splunkd_uri=self.make_uri(ep_id),
+                    session_key=self.session_key,
+                    data=data,
+                    retry=3
+                )
+
+                if resp is None or resp.status != 200:
+                    msg = 'Fail to load endpoint "{ep_id}" - {err}' \
+                          ''.format(ep_id=ep_id,
+                                    err=code_to_msg(resp, cont)
+                                    if resp else cont)
+                    log(msg, level=logging.ERROR, need_tb=True)
+                    raise ConfigException(msg)
+
+                try:
+                    ret[ep_id] = self._parse_content(ep_id, cont)
+                except ConfigException, exc:
+                    log(exc, level=logging.WARNING, need_tb=True)
+                    if retry < retries-1:
+                        time.sleep(waiting_time[retry])
+                else:
+                    break
+            else:
+                log(exc, level=logging.ERROR, need_tb=True)
+                raise exc
+
+        log('"load" method out', level=logging.DEBUG)
+        return ret
+
+    def update_items(self, endpoint_id, item_names, field_names, data,
+                     raise_if_failed=False):
+        """Update items in specified endpoint with given fields in data
+        :param endpoint_id: endpoint id in schema, the key name in schema
+        :param item_names: a list of item name
+        :param field_names: a list of updated fields
+        :param data: a dict of content for items, for example:
+            {
+                "item_name_1": {
+                    "field_name_1": "value_1",
+                    "field_name_2": "value_2",
+                },
+                "item_name_2": {
+                    "field_name_1": "value_1x",
+                    "field_name_2": "value_2x",
+                }
+            }
+        :raise_if_failed: raise an exception if updating failed.
+        :return: a list of endpoint ids, which are failed to be updated.
+            If raise_if_failed is True, it will exist with an exception
+            on any updating failed.
+        """
+        log('"update_items" method in',
+            msgx='endpoint_id=%s, item_names=%s, field_names=%s'
+                 % (endpoint_id, item_names, field_names),
+            level=logging.DEBUG)
+
+        assert endpoint_id in self._endpoints, \
+            'Unexpected endpoint id in given schema - {ep_id}' \
+            ''.format(ep_id=endpoint_id)
+
+        item_names_failed = []
+        for item_name in item_names:
+            item_data = data.get(item_name, {})
+            item_data = {field_name: self.dump_value(endpoint_id,
+                                                     item_name,
+                                                     field_name,
+                                                     item_data[field_name])
+                         for field_name in field_names
+                         if field_name in item_data}
+            if not item_data:
+                continue
+            item_uri = self.make_uri(endpoint_id, item_name=item_name)
+
+            resp, cont = splunkd_request(splunkd_uri=item_uri,
+                                         session_key=self.session_key,
+                                         data=item_data,
+                                         method="POST",
+                                         retry=3
+                                         )
+            if resp is None or resp.status not in (200, 201):
+                msg = 'Fail to update item "{item}" in endpoint "{ep_id}"' \
+                      ' - {err}'.format(ep_id=endpoint_id,
+                                        item=item_name,
+                                        err=code_to_msg(resp, cont)
+                                        if resp else cont)
+                log(msg, level=logging.ERROR)
+                if raise_if_failed:
+                    raise ConfigException(msg)
+                item_names_failed.append(item_name)
+
+        log('"update_items" method out', level=logging.DEBUG)
+        return item_names_failed
+
+    def make_uri(self, endpoint_id, item_name=None):
+        """Make uri for REST endpoint in TA according to given schema
+        :param endpoint_id: endpoint id in schema
+        :param item_name: item name for given endpoint. None for listing all
+        :return:
+        """
+        endpoint = self._endpoints[endpoint_id]['endpoint']
+        ep_full = endpoint[1:].strip('/') \
+            if endpoint.startswith(Config.NON_PROC_ENDPOINT) else \
+            '{admin_match}/{protocol_version}/{endpoint}' \
+            ''.format(admin_match=self._rest_namespace,
+                      protocol_version=self._protocol_version,
+                      endpoint=(self._rest_prefix +
+                                self._endpoints[endpoint_id]['endpoint']))
+        ep_uri = None if endpoint_id not in self._endpoints else \
+            '{splunkd_uri}/servicesNS/{user}/{app}/{endpoint_full}' \
+            ''.format(splunkd_uri=self.splunkd_uri,
+                      user=self.user,
+                      app=self.app,
+                      endpoint_full=ep_full
+                      )
+
+        url = ep_uri if item_name is None else "{ep_uri}/{item_name}"\
+            .format(ep_uri=ep_uri, item_name=quote(item_name))
+        if item_name is None:
+            url += '?count=-1'
+        log('"make_uri" method', msgx='url=%s' % url,
+            level=logging.DEBUG)
+        return url
+
+    def _parse_content(self, endpoint_id, content):
+        """Parse content returned from REST
+        :param content: a JSON string returned from REST.
+        """
+        try:
+            content = json.loads(content)['entry']
+            ret = {ent['name']: ent['content'] for ent in content}
+        except Exception as exc:
+            msg = 'Fail to parse content from endpoint_id=%s' \
+                  ' - %s' % (endpoint_id, exc)
+            log(msg, level=logging.ERROR, need_tb=True)
+            raise ConfigException(msg)
+
+        ret = {name: {key: self.load_value(endpoint_id, name, key, val)
+                      for key, val in ent.iteritems()
+                      if not key.startswith('eai:')}
+               for name, ent in ret.iteritems()}
+        return ret
+
+    def _parse_schema(self, ucc_config_schema):
+        try:
+            ucc_config_schema = json.loads(ucc_config_schema)
+        except ValueError:
+            msg = 'Invalid JSON content of schema'
+            log(msg, level=logging.ERROR, need_tb=True)
+            raise ConfigException(msg)
+        except Exception as exc:
+            log(exc, level=logging.ERROR, need_tb=True)
+            raise ConfigException(exc)
+
+        ucc_config_schema.update({key: val for key, val in
+                                  Config.META_FIELDS_DEFAULT.iteritems()
+                                  if key not in ucc_config_schema})
+        for field in Config.META_FIELDS:
+            assert field in ucc_config_schema and \
+                isinstance(ucc_config_schema[field], basestring), \
+                'Missing or invalid field "%s" in given schema' % field
+            setattr(self, field, ucc_config_schema[field])
+
+        self._endpoints = {}
+        for key, val in ucc_config_schema.iteritems():
+            if key.startswith('_'):
+                continue
+
+            assert isinstance(val, dict), \
+                'The schema of endpoint "%s" should be dict' % key
+            assert 'endpoint' in val, \
+                'The endpoint "%s" has no endpoint entry' % key
+
+            self._endpoints[key] = val
+
+    def _check_protocol_version(self):
+        """
+        Check if the protocol version in given schema is supported.
+        :return:
+        """
+        if not self._protocol_version:
+            return
+        if not self._protocol_version.startswith('1.'):
+            raise ConfigException('Unsupported protocol version "%s" '
+                                  'in given schema' % self._protocol_version)
+
+    def load_value(self, endpoint_id, item_name, fname, fval):
+        field_type = self._get_field_type(endpoint_id, item_name, fname)
+        if field_type == '':
+            return fval
+
+        try:
+            field_type = field_type.lower()
+            if field_type == 'bool':
+                return True if sc_util.is_true(fval) else False
+            elif field_type == 'int':
+                return int(fval)
+            elif field_type == 'json':
+                return json.loads(fval)
+        except Exception as exc:
+            msg = 'Fail to load value of "{type_name}" - ' \
+                  'endpoint={endpoint}, item={item}, field={field}' \
+                  ''.format(type_name=field_type,
+                            endpoint=endpoint_id,
+                            item=item_name,
+                            field=fname)
+            log(msg, msgx=str(exc), level=logging.WARNING, need_tb=True)
+            raise ConfigException(msg)
+
+    def dump_value(self, endpoint_id, item_name, fname, fval):
+        field_type = self._get_field_type(endpoint_id, item_name, fname)
+        if field_type == '':
+            return fval
+
+        try:
+            field_type = field_type.lower()
+            if field_type == 'bool':
+                return str(fval).lower()
+            elif field_type == 'json':
+                return json.dumps(fval)
+            else:
+                return fval
+        except Exception, exc:
+            msg = 'Fail to dump value of "{type_name}" - ' \
+                  'endpoint={endpoint}, item={item}, field={field}' \
+                  ''.format(type_name=field_type,
+                            endpoint=endpoint_id,
+                            item=item_name,
+                            field=fname)
+            log(msg, msgx=str(exc), level=logging.ERROR, need_tb=True)
+            raise ConfigException(msg)
+
+    def _get_field_type(self, endpoint_id, item_name, fname):
+        field_types = self._endpoints[endpoint_id].get('field_types', {})
+        if item_name in field_types:
+            fields = field_types[item_name]
+        elif Config.FIELD_PLACEHOLDER in field_types:
+            fields = field_types[Config.FIELD_PLACEHOLDER]
+        else:
+            fields = {}
+
+        field_type = fields.get(fname, '')
+        if field_type not in ('', 'bool', 'int', 'json'):
+            msg = 'Unsupported type "{type_name}" for value in schema - ' \
+                  'endpoint={endpoint}, item={item}, field={field}' \
+                  ''.format(type_name=field_type,
+                            endpoint=endpoint_id,
+                            item=item_name,
+                            field=fname)
+            log(msg, level=logging.ERROR, need_tb=True)
+            raise ConfigException(msg)
+        return field_type
+
+    def get_endpoints(self):
+        return self._endpoints

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.2"

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_checkpoint_manager.py

@@ -0,0 +1,147 @@
+import json
+import re
+
+from . import ta_consts as c
+from . import ta_helper as th
+from ..common import log as stulog
+from ...splunktalib import state_store as ss
+from ...splunktalib.common.util import is_true
+
+
+class TACheckPointMgr(object):
+    SEPARATOR = "_" * 3
+
+    # FIXME We'd better move all default values together
+    _DEFAULT_MAX_CACHE_SECONDS = 5
+    _MAXIMUM_MAX_CACHE_SECONDS = 3600
+
+    def __init__(self, meta_config, task_config):
+        self._task_config = task_config
+        self._store = self._create_state_store(
+            meta_config,
+            task_config.get(c.checkpoint_storage_type),
+            task_config[c.appname]
+        )
+
+    def _create_state_store(self, meta_config, storage_type, app_name):
+        stulog.logger.debug('Got checkpoint storage type=%s', storage_type)
+
+        if storage_type == c.checkpoint_kv_storage:
+            collection_name = self._get_collection_name()
+            stulog.logger.debug(
+                'Creating KV state store, collection name=%s', collection_name
+            )
+            return ss.get_state_store(
+                meta_config,
+                appname=app_name,
+                collection_name=collection_name,
+                use_kv_store=True
+            )
+
+        use_cache_file = self._use_cache_file()
+        max_cache_seconds = \
+            self._get_max_cache_seconds() if use_cache_file else None
+
+        stulog.logger.debug(
+            'Creating file state store, use_cache_file=%s, max_cache_seconds=%s',
+            use_cache_file, max_cache_seconds
+        )
+
+        return ss.get_state_store(
+            meta_config,
+            app_name,
+            use_cache_file=use_cache_file,
+            max_cache_seconds=max_cache_seconds
+        )
+
+    def _get_collection_name(self):
+        collection = self._task_config.get(c.collection_name)
+        collection = collection.strip() if collection else ''
+
+        if not collection:
+            input_name = self._task_config[c.mod_input_name]
+            stulog.logger.info(
+                'Collection name="%s" is empty, set it to "%s"',
+                collection, input_name
+            )
+            collection = input_name
+        return re.sub(r'[^\w]+', '_', collection)
+
+    def _use_cache_file(self):
+        # TODO Move the default value outside code
+        use_cache_file = is_true(self._task_config.get(c.use_cache_file, True))
+        if use_cache_file:
+            stulog.logger.info(
+                "Stanza=%s using cached file store to create checkpoint",
+                self._task_config[c.stanza_name]
+            )
+        return use_cache_file
+
+    def _get_max_cache_seconds(self):
+        default = self._DEFAULT_MAX_CACHE_SECONDS
+        seconds = self._task_config.get(
+            c.max_cache_seconds, default
+        )
+        try:
+            seconds = int(seconds)
+        except ValueError:
+            stulog.logger.warning(
+                "The max_cache_seconds '%s' is not a valid integer,"
+                " so set this variable to default value %s",
+                seconds, default
+            )
+            seconds = default
+        else:
+            maximum = self._MAXIMUM_MAX_CACHE_SECONDS
+            if not (1 <= seconds <= maximum):
+                # for seconds>3600 set it to 3600. for seconds <=0 set it to default.
+                adjusted = max(min(seconds, maximum), default)
+                stulog.logger.warning(
+                    "The max_cache_seconds (%s) is expected in range[1,%s],"
+                    " set it to %s",
+                    seconds, maximum, adjusted
+                )
+                seconds = adjusted
+        return seconds
+
+    def get_ckpt_key(self, namespaces=None):
+        return self._key_formatter(namespaces)
+
+    def get_ckpt(self, namespaces=None, show_namespaces=False):
+        key, namespaces = self.get_ckpt_key(namespaces)
+        raw_checkpoint = self._store.get_state(key)
+        stulog.logger.info("Get checkpoint key='%s' value='%s'",
+                           key, json.dumps(raw_checkpoint))
+        if not show_namespaces and raw_checkpoint:
+            return raw_checkpoint.get("data")
+        return raw_checkpoint
+
+    def update_ckpt(self, ckpt, namespaces=None):
+        if not ckpt:
+            stulog.logger.warning("Checkpoint expect to be not empty.")
+            return
+        key, namespaces = self.get_ckpt_key(namespaces)
+        value = {"namespaces": namespaces, "data": ckpt}
+        stulog.logger.info("Update checkpoint key='%s' value='%s'",
+                           key, json.dumps(value))
+        self._store.update_state(key, value)
+
+    def remove_ckpt(self, namespaces=None):
+        key, namespaces = self.get_ckpt_key(namespaces)
+        self._store.delete_state(key)
+
+    def _key_formatter(self, namespaces=None):
+        if not namespaces:
+            stulog.logger.info('Namespaces is empty, using stanza name instead.')
+            namespaces = [self._task_config[c.stanza_name]]
+        key_str = TACheckPointMgr.SEPARATOR.join(namespaces)
+        hashed_file = th.format_name_for_file(key_str)
+        stulog.logger.info("raw_file='%s' hashed_file='%s'", key_str, hashed_file)
+        return hashed_file, namespaces
+
+    def close(self, key=None):
+        try:
+            self._store.close(key)
+            stulog.logger.info('Closed state store successfully. key=%s', key)
+        except Exception:
+            stulog.logger.exception('Error closing state store. key=%s', key)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_config.py

@@ -0,0 +1,155 @@
+import os.path as op
+import socket
+
+import ta_consts as c
+import ta_helper as th
+from ..common import log as stulog
+from ...splunktalib import modinput as modinput
+from ...splunktalib import splunk_cluster as sc
+from ...splunktalib.common import util
+
+
+# methods can be overrided by subclass : process_task_configs
+class TaConfig(object):
+    _current_hostname = socket.gethostname()
+    _appname = util.get_appname_from_path(op.abspath(__file__))
+
+    def __init__(self, meta_config, client_schema, log_suffix=None,
+                 stanza_name=None, input_type=None,
+                 single_instance=True):
+        self._meta_config = meta_config
+        self._stanza_name = stanza_name
+        self._input_type = input_type
+        self._log_suffix = log_suffix
+        self._single_instance = single_instance
+        self._task_configs = []
+        self._client_schema = client_schema
+        self._server_info = sc.ServerInfo(meta_config[c.server_uri],
+                                          meta_config[c.session_key])
+        self._all_conf_contents = {}
+        self._get_division_settings = {}
+        self.set_logging()
+        self._load_task_configs()
+
+    def is_shc_member(self):
+        return self._server_info.is_shc_member()
+
+    def is_search_head(self):
+        return self._server_info.is_search_head()
+
+    def is_single_instance(self):
+        return self._single_instance
+
+    def get_meta_config(self):
+        return self._meta_config
+
+    def get_task_configs(self):
+        return self._task_configs
+
+    def get_all_conf_contents(self):
+        if self._all_conf_contents:
+            return self._all_conf_contents.get(c.inputs), \
+                   self._all_conf_contents.get(c.all_configs), \
+                   self._all_conf_contents.get(c.global_settings)
+
+        inputs, configs, global_settings = th.get_all_conf_contents(
+            self._meta_config[c.server_uri],
+            self._meta_config[c.session_key],
+            self._client_schema, self._input_type)
+        self._all_conf_contents[c.inputs] = inputs
+        self._all_conf_contents[c.all_configs] = configs
+        self._all_conf_contents[c.global_settings] = global_settings
+        return inputs, configs, global_settings
+
+    def set_logging(self):
+        # The default logger name is "cloud_connect_engine"
+        if self._stanza_name and self._log_suffix:
+            logger_name = self._log_suffix + "_" + th.format_name_for_file(
+                self._stanza_name)
+            stulog.reset_logger(logger_name)
+        inputs, configs, global_settings = self.get_all_conf_contents()
+        log_level = "INFO"
+        for item in global_settings.get("settings"):
+            if item.get(c.name) == "logging" and item.get("loglevel"):
+                log_level = item["loglevel"]
+                break
+        stulog.set_log_level(log_level)
+        stulog.logger.info("Set log_level={}".format(log_level))
+        stulog.logger.info("Start {} task".format(self._stanza_name))
+
+    def get_input_type(self):
+        return self._input_type
+
+    def _get_checkpoint_storage_type(self, config):
+        cs_type = config.get(c.checkpoint_storage_type)
+        stulog.logger.debug("Checkpoint storage type=%s", cs_type)
+
+        cs_type = cs_type.strip() if cs_type else c.checkpoint_auto
+
+        # Allow user configure 'auto' and 'file' only.
+        if cs_type not in (c.checkpoint_auto, c.checkpoint_file):
+            stulog.logger.warning(
+                "Checkpoint storage type='%s' is invalid, change it to '%s'",
+                cs_type, c.checkpoint_auto
+            )
+            cs_type = c.checkpoint_auto
+
+        if cs_type == c.checkpoint_auto and self.is_search_head():
+            stulog.logger.info(
+                "Checkpoint storage type is '%s' and instance is "
+                "search head, set checkpoint storage type to '%s'.",
+                c.checkpoint_auto,
+                c.checkpoint_kv_storage
+            )
+            cs_type = c.checkpoint_kv_storage
+        return cs_type
+
+    def _load_task_configs(self):
+        inputs, configs, global_settings = self.get_all_conf_contents()
+        if self._input_type:
+            inputs = inputs.get(self._input_type)
+        if not self._single_instance:
+            inputs = [input for input in inputs if
+                      input[c.name] == self._stanza_name]
+        all_task_configs = []
+        for input in inputs:
+            task_config = {}
+            task_config.update(input)
+            task_config[c.configs] = configs
+            task_config[c.settings] = \
+                {item[c.name]: item for item in global_settings["settings"]}
+            if self.is_single_instance():
+                collection_interval = "collection_interval"
+                task_config[c.interval] = task_config.get(collection_interval)
+            task_config[c.interval] = int(task_config[c.interval])
+            if task_config[c.interval] <= 0:
+                raise ValueError(
+                    "The interval value {} is invalid."
+                    " It should be a positive integer".format(
+                        task_config[c.interval]))
+
+            task_config[c.checkpoint_storage_type] = \
+                self._get_checkpoint_storage_type(task_config)
+
+            task_config[c.appname] = TaConfig._appname
+            task_config[c.mod_input_name] = self._input_type
+            task_config[c.stanza_name] = task_config[c.name]
+
+            all_task_configs.append(task_config)
+        self._task_configs = all_task_configs
+
+    # Override this method if some transforms or validations needs to be done
+    # before task_configs is exposed
+    def process_task_configs(self, task_configs):
+        pass
+
+
+def create_ta_config(settings, config_cls=TaConfig, log_suffix=None,
+                     single_instance=True):
+    meta_config, configs = modinput.get_modinput_configs_from_stdin()
+    stanza_name = None
+    input_type = None
+    if configs and "://" in configs[0].get("name", ""):
+        input_type, stanza_name = configs[0].get("name").split("://", 1)
+    return config_cls(meta_config, settings, log_suffix, stanza_name,
+                      input_type, single_instance=single_instance)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_consts.py

@@ -0,0 +1,54 @@
+server_uri = "server_uri"
+session_key = "session_key"
+version = "version"
+appname = "appname"
+event_writer = "event_writer"
+index = "index"
+default_index = "default"
+source = "source"
+sourcetype = "sourcetype"
+data_loader = "data_loader"
+
+meta_configs = "meta_configs"
+disabled = "disabled"
+resource = "resource"
+events = "events"
+scope = "scope"
+checkpoint_dir = "checkpoint_dir"
+
+ckpt_dict = "ckpt_dict"
+inputs = "inputs"
+input_name = "input_name"
+input_data = "input_data"
+interval = "interval"
+data = "data"
+batch_size = 'batch_size'
+time_fmt = "%Y-%m-%dT%H:%M:%S"
+utc_time_fmt = "%Y-%m-%dT%H:%M:%S.%fZ"
+
+# system setting keys
+checkpoint_storage_type = "builtin_system_checkpoint_storage_type"
+
+# Possible values for checkpoint storage type
+checkpoint_auto = 'auto'
+checkpoint_kv_storage = 'kv_store'
+checkpoint_file = 'file'
+
+# For cache file
+use_cache_file = "builtin_system_use_cache_file"
+max_cache_seconds = "builtin_system_max_cache_seconds"
+# For kv store
+collection_name = "builtin_system_kvstore_collection_name"
+
+settings = "__settings__"
+configs = "__configs__"
+
+name = "name"
+config = "config"
+division = "division"
+stanza_name = "stanza_name"
+divide_key = "_divide_key"
+divide_endpoint = "_divide_endpoint"
+mod_input_name = "mod_input_name"
+global_settings = "global_settings"
+all_configs = "all_configs"

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_data_client.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python
+from . import ta_checkpoint_manager as cp
+from . import ta_data_collector as tdc
+
+
+def build_event(host=None,
+                source=None,
+                sourcetype=None,
+                time=None,
+                index=None,
+                raw_data="",
+                is_unbroken=False,
+                is_done=False):
+    if is_unbroken is False and is_done is True:
+        raise Exception('is_unbroken=False is_done=True is invalid')
+    return tdc.event_tuple._make([host, source, sourcetype, time, index,
+                                  raw_data, is_unbroken, is_done])
+
+
+class TaDataClient(object):
+    def __init__(self,
+                 meta_config,
+                 task_config,
+                 checkpoint_mgr=None,
+                 event_writer=None):
+        self._meta_config = meta_config
+        self._task_config = task_config
+        self._checkpoint_mgr = checkpoint_mgr
+        self._event_writer = event_writer
+        self._stop = False
+
+    def is_stopped(self):
+        return self._stop
+
+    def stop(self):
+        self._stop = True
+
+    def get(self):
+        raise StopIteration
+
+
+def create_data_collector(dataloader,
+                          tconfig,
+                          meta_configs,
+                          task_config,
+                          data_client_cls,
+                          checkpoint_cls=None):
+    checkpoint_manager_cls = checkpoint_cls or cp.TACheckPointMgr
+    return tdc.TADataCollector(tconfig, meta_configs, task_config,
+                               checkpoint_manager_cls, data_client_cls,
+                               dataloader)
+
+
+def client_adatper(job_func):
+    class TaDataClientAdapter(TaDataClient):
+        def __init__(self, all_conf_contents, meta_config, task_config,
+                     chp_mgr):
+            super(TaDataClientAdapter, self).__init__(meta_config, task_config,
+                                                      chp_mgr)
+            self._execute_times = 0
+            self._gen = job_func(self._task_config, chp_mgr)
+
+        def stop(self):
+            """
+            overwrite to handle stop control command
+            """
+
+            # normaly base class just set self._stop as True
+            super(TaDataClientAdapter, self).stop()
+
+        def get(self):
+            """
+            overwrite to get events
+            """
+            self._execute_times += 1
+            if self.is_stopped():
+                # send stop signal
+                self._gen.send(self.is_stopped())
+                raise StopIteration
+            if self._execute_times == 1:
+                return self._gen.next()
+            return self._gen.send(self.is_stopped())
+
+    return TaDataClientAdapter

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_data_collector.py

@@ -0,0 +1,144 @@
+#!/usr/bin/python
+import threading
+import time
+from collections import namedtuple
+
+import ta_consts as c
+from ..common import log as stulog
+from ...splunktalib.common import util as scu
+
+evt_fmt = ("<stream><event><host>{0}</host>"
+           "<source><![CDATA[{1}]]></source>"
+           "<sourcetype><![CDATA[{2}]]></sourcetype>"
+           "<time>{3}</time>"
+           "<index>{4}</index><data>"
+           "<![CDATA[{5}]]></data></event></stream>")
+
+unbroken_evt_fmt = ("<stream>"
+                    "<event unbroken=\"1\">"
+                    "<host>{0}</host>"
+                    "<source><![CDATA[{1}]]></source>"
+                    "<sourcetype><![CDATA[{2}]]></sourcetype>"
+                    "<time>{3}</time>"
+                    "<index>{4}</index>"
+                    "<data><![CDATA[{5}]]></data>"
+                    "{6}"
+                    "</event>"
+                    "</stream>")
+
+event_tuple = namedtuple('Event',
+                         ['host', 'source', 'sourcetype', 'time', 'index',
+                          'raw_data', 'is_unbroken', 'is_done'])
+
+
+class TADataCollector(object):
+    def __init__(self, tconfig, meta_config, task_config,
+                 checkpoint_manager_cls, data_client_cls, data_loader):
+        self._lock = threading.Lock()
+        self._ta_config = tconfig
+        self._meta_config = meta_config
+        self._task_config = task_config
+        self._stopped = False
+        self._p = self._get_logger_prefix()
+        self._checkpoint_manager = checkpoint_manager_cls(meta_config,
+                                                          task_config)
+        self.data_client_cls = data_client_cls
+        self._data_loader = data_loader
+        self._client = None
+
+    def get_meta_configs(self):
+        return self._meta_config
+
+    def get_task_config(self):
+        return self._task_config
+
+    def get_interval(self):
+        return self._task_config[c.interval]
+
+    def _get_logger_prefix(self):
+        pairs = ['{}="{}"'.format(c.stanza_name, self._task_config[
+            c.stanza_name])]
+        return "[{}]".format(" ".join(pairs))
+
+    def stop(self):
+        self._stopped = True
+        if self._client:
+            self._client.stop()
+
+    def __call__(self):
+        self.index_data()
+
+    def _build_event(self, events):
+        if not events:
+            return None
+        if not isinstance(events, list):
+            events = [events]
+        evts = []
+        for event in events:
+            assert event.raw_data, "the raw data of events is empty"
+            if event.is_unbroken:
+                evt = unbroken_evt_fmt.format(
+                    event.host or "", event.source or "", event.sourcetype or
+                    "", event.time or "", event.index or "",
+                    scu.escape_cdata(event.raw_data), "<done/>" if
+                    event.is_done else "")
+            else:
+                evt = evt_fmt.format(event.host or "", event.source or "",
+                                     event.sourcetype or "", event.time or "",
+                                     event.index or "",
+                                     scu.escape_cdata(event.raw_data))
+            evts.append(evt)
+        return evts
+
+    def _create_data_client(self):
+        return self.data_client_cls(self._meta_config,
+                                    self._task_config,
+                                    self._checkpoint_manager,
+                                    self._data_loader.get_event_writer())
+
+    def index_data(self):
+        if self._lock.locked():
+            stulog.logger.debug(
+                "Last round of stanza={} is not done yet".format(
+                    self._task_config[c.stanza_name]))
+            return
+        with self._lock:
+            try:
+                self._do_safe_index()
+                self._checkpoint_manager.close()
+            except Exception:
+                stulog.logger.exception("{} Failed to index data"
+                                        .format(self._p))
+            stulog.logger.info("{} End of indexing data".format(self._p))
+            if not self._ta_config.is_single_instance():
+                self._data_loader.tear_down()
+
+    def _write_events(self, events):
+        evts = self._build_event(events)
+        if evts:
+            if not self._data_loader.write_events(evts):
+                stulog.logger.info("{} the event queue is closed and the "
+                                   "received data will be discarded".format(
+                                       self._p))
+                return False
+        return True
+
+    def _do_safe_index(self):
+        self._client = self._create_data_client()
+        while not self._stopped:
+            try:
+                events = self._client.get()
+                if not events:
+                    continue
+                else:
+                    if not self._write_events(events):
+                        break
+            except StopIteration:
+                stulog.logger.info("{} Finished this round".format(self._p))
+                return
+            except Exception:
+                stulog.logger.exception("{} Failed to get msg".format(self._p))
+                break
+        # in case encounter exception or fail to write events
+        if not self._stopped:
+            self.stop()

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_data_loader.py

@@ -0,0 +1,168 @@
+"""
+Data Loader main entry point
+"""
+
+
+import Queue
+import os.path as op
+import ConfigParser
+
+from ...splunktalib.concurrent import concurrent_executor as ce
+from ...splunktalib import timer_queue as tq
+from ...splunktalib.schedule import job as sjob
+from ...splunktalib.common import log
+
+
+class TADataLoader(object):
+    """
+    Data Loader boots all underlying facilities to handle data collection
+    """
+
+    def __init__(self, job_scheduler, event_writer):
+        """
+        @configs: a list like object containing a list of dict
+        like object. Each element shall implement dict.get/[] like interfaces
+        to get the value for a key.
+        @job_scheduler: schedulering the jobs. shall implement get_ready_jobs
+        @event_writer: write_events
+        """
+
+        self._settings = self._read_default_settings()
+        self._settings["daemonize_thread"] = False
+        self._event_writer = event_writer
+        self._wakeup_queue = Queue.Queue()
+        self._scheduler = job_scheduler
+        self._timer_queue = tq.TimerQueue()
+        self._executor = ce.ConcurrentExecutor(self._settings)
+        self._started = False
+
+    def run(self, jobs):
+        if self._started:
+            return
+        self._started = True
+
+        self._event_writer.start()
+        self._executor.start()
+        self._timer_queue.start()
+        self._scheduler.start()
+        log.logger.info("TADataLoader started.")
+
+        def _enqueue_io_job(job):
+            job_props = job.get_props()
+            real_job = job_props["real_job"]
+            self.run_io_jobs((real_job,))
+
+        for job in jobs:
+            j = sjob.Job(_enqueue_io_job, {"real_job": job},
+                         job.get_interval())
+            self._scheduler.add_jobs((j,))
+
+        self._wait_for_tear_down()
+
+        for job in jobs:
+            job.stop()
+
+        self._scheduler.tear_down()
+        self._timer_queue.tear_down()
+        self._executor.tear_down()
+        self._event_writer.tear_down()
+        log.logger.info("DataLoader stopped.")
+
+    def _wait_for_tear_down(self):
+        wakeup_q = self._wakeup_queue
+        while 1:
+            try:
+                go_exit = wakeup_q.get(timeout=1)
+            except Queue.Empty:
+                pass
+            else:
+                if go_exit:
+                    log.logger.info("DataLoader got stop signal")
+                    self._stopped = True
+                    break
+
+    def tear_down(self):
+        self._wakeup_queue.put(True)
+        log.logger.info("DataLoader is going to stop.")
+
+    def stopped(self):
+        return self._stopped
+
+    def run_io_jobs(self, jobs, block=True):
+        self._executor.enqueue_io_funcs(jobs, block)
+
+    def run_compute_job(self, func, args=(), kwargs={}):
+        self._executor.run_compute_func_sync(func, args, kwargs)
+
+    def run_compute_job_async(self, func, args=(), kwargs={}, callback=None):
+        """
+        @return: AsyncResult
+        """
+
+        return self._executor.run_compute_func_async(func, args,
+                                                     kwargs, callback)
+
+    def add_timer(self, callback, when, interval):
+        return self._timer_queue.add_timer(callback, when, interval)
+
+    def remove_timer(self, timer):
+        self._timer_queue.remove_timer(timer)
+
+    def write_events(self, events):
+        return self._event_writer.write_events(events)
+
+    def get_event_writer(self):
+        return self._event_writer
+
+    @staticmethod
+    def _read_default_settings():
+        cur_dir = op.dirname(op.abspath(__file__))
+        setting_file = op.join(cur_dir,"../../","splunktalib", "setting.conf")
+        parser = ConfigParser.ConfigParser()
+        parser.read(setting_file)
+        settings = {}
+        keys = ("process_size", "thread_min_size", "thread_max_size",
+                "task_queue_size")
+        for option in keys:
+            try:
+                settings[option] = parser.get("global", option)
+            except ConfigParser.NoOptionError:
+                settings[option] = -1
+
+            try:
+                settings[option] = int(settings[option])
+            except ValueError:
+                settings[option] = -1
+        log.logger.debug("settings: %s", settings)
+        return settings
+
+
+class GlobalDataLoader(object):
+    """ Singleton, inited when started"""
+
+    __instance = None
+
+    @staticmethod
+    def get_data_loader(scheduler, writer):
+        if GlobalDataLoader.__instance is None:
+            GlobalDataLoader.__instance = TADataLoader(
+                scheduler, writer)
+        return GlobalDataLoader.__instance
+
+    @staticmethod
+    def reset():
+        GlobalDataLoader.__instance = None
+
+
+def create_data_loader():
+    """
+    create a data loader with default event_writer, job_scheudler
+    """
+
+    from ...splunktalib import event_writer as ew
+    from ...splunktalib.schedule import scheduler as sched
+
+    writer = ew.EventWriter()
+    scheduler = sched.Scheduler()
+    loader = GlobalDataLoader.get_data_loader(scheduler, writer)
+    return loader

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_helper.py

@@ -0,0 +1,154 @@
+import hashlib
+import json
+import os.path as op
+import re
+from calendar import timegm
+from datetime import datetime
+
+import functools32
+from splunktaucclib.global_config import GlobalConfig, GlobalConfigSchema
+from . import ta_consts as c
+from ...splunktacollectorlib import config as sc
+from ...splunktalib.common import util
+
+
+def utc2timestamp(human_time):
+    regex1 = ur"\d{4}-\d{2}-\d{2}.\d{2}:\d{2}:\d{2}"
+    match = re.search(regex1, human_time)
+    if match:
+        formated = match.group()
+    else:
+        return None
+
+    strped_time = datetime.strptime(formated, c.time_fmt)
+    timestamp = timegm(strped_time.utctimetuple())
+
+    regex2 = "\d{4}-\d{2}-\d{2}.\d{2}:\d{2}:\d{2}(\.\d+)"
+    match = re.search(regex2, human_time)
+    if match:
+        timestamp += float(match.group(1))
+    else:
+        timestamp += float("0.000000")
+    return timestamp
+
+
+def get_md5(data):
+    """
+    function name is not change, actually use sha1 instead
+    :param data:
+    :return:
+    """
+    assert data is not None, "The input cannot be None"
+    if isinstance(data, (unicode, str)):
+        return hashlib.sha256(data.encode('utf-8')).hexdigest()
+    elif isinstance(data, (list, tuple, dict)):
+        return hashlib.sha256(json.dumps(data).encode('utf-8')).hexdigest()
+
+
+def get_all_conf_contents(server_uri, sessionkey, settings, input_type=None):
+    schema = GlobalConfigSchema(settings)
+    global_config = GlobalConfig(
+        server_uri, sessionkey, schema
+    )
+    inputs = global_config.inputs.load(input_type=input_type)
+    configs = global_config.configs.load()
+    settings = global_config.settings.load()
+    return inputs, configs, settings
+
+
+@functools32.lru_cache(maxsize=64)
+def format_name_for_file(name):
+    return hashlib.sha256(name).hexdigest()
+
+
+class ConfigSchemaHandler(object):
+    _app_name = util.get_appname_from_path(op.abspath(__file__))
+    # Division schema keys.
+    TYPE = "type"
+
+    TYPE_SINGLE = "single"
+    TYPE_MULTI = "multi"
+    REFER = "refer"
+    SEPARATOR = "separator"
+
+    def __init__(self, meta_configs, client_schema):
+        self._config = sc.Config(splunkd_uri=meta_configs[c.server_uri],
+                                 session_key=meta_configs[c.session_key],
+                                 schema=json.dumps(client_schema[
+                                                       c.config]),
+                                 user="nobody",
+                                 app=ConfigSchemaHandler._app_name)
+        self._client_schema = client_schema
+        self._all_conf_contents = {}
+        self._load_conf_contents()
+        self._division_settings = self._divide_settings()
+
+    def get_endpoints(self):
+        return self._config.get_endpoints()
+
+    def get_all_conf_contents(self):
+        return self._all_conf_contents
+
+    def get_single_conf_contents(self, endpoint):
+        return self._all_conf_contents.get(endpoint)
+
+    def get_division_settings(self):
+        return self._division_settings
+
+    def _divide_settings(self):
+        division_schema = self._client_schema[c.division]
+        division_settings = dict()
+        for division_endpoint, division_contents in division_schema.iteritems():
+            division_settings[division_endpoint] = self._process_division(
+                division_endpoint, division_contents)
+        return division_settings
+
+    def _load_conf_contents(self):
+        self._all_conf_contents = self._config.load()
+
+    def _process_division(self, division_endpoint, division_contents):
+        division_metrics = []
+        assert isinstance(division_contents, dict)
+        for division_key, division_value in division_contents.iteritems():
+            try:
+                assert self.TYPE in division_value and \
+                       division_value[self.TYPE] in \
+                       [self.TYPE_SINGLE, self.TYPE_MULTI] and \
+                       self.SEPARATOR in division_value if \
+                    division_value[self.TYPE] == self.TYPE_MULTI else True
+            except Exception:
+                raise Exception("Invalid division schema")
+            division_metrics.append(DivisionRule(division_endpoint,
+                                                 division_key,
+                                                 division_value[self.TYPE],
+                                                 division_value.get(
+                                                     self.SEPARATOR,
+                                                 ),
+                                                 division_value.get(
+                                                     self.REFER,
+                                                 )))
+        return division_metrics
+
+
+class DivisionRule(object):
+    def __init__(self, endpoint, metric, type, separator, refer):
+        self._endpoint = endpoint
+        self._metric = metric
+        self._type = type
+        self._separator = separator
+        self._refer = refer
+
+    def endpoint(self):
+        return self._endpoint
+
+    def metric(self):
+        return self._metric
+
+    def type(self):
+        return self._type
+
+    def separator(self):
+        return self._separator
+
+    def refer(self):
+        return self._refer

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_mod_input.py

@@ -0,0 +1,272 @@
+#!/usr/bin/python
+
+"""
+This is the main entry point for My TA
+"""
+
+import os.path as op
+import platform
+import sys
+import time
+
+from . import ta_checkpoint_manager as cpmgr
+from . import ta_config as tc
+from . import ta_data_client as tdc
+from . import ta_data_loader as dl
+from ..common import load_schema_file as ld
+from ..common import log as stulog
+from ...common.lib_util import get_app_root_dir, get_mod_input_script_name
+from ...splunktalib import file_monitor as fm
+from ...splunktalib import modinput
+from ...splunktalib import orphan_process_monitor as opm
+from ...splunktalib.common import util as utils
+
+utils.remove_http_proxy_env_vars()
+
+__CHECKPOINT_DIR_MAX_LEN__ = 180
+
+
+def do_scheme(
+        mod_input_name,
+        schema_para_list=None,
+        single_instance=True,
+):
+    """
+    Feed splunkd the TA's scheme
+
+    """
+    builtin_names = {
+        "name", "index", "sourcetype", "host", "source",
+        "disabled", "interval"
+    }
+
+    param_string_list = []
+    if schema_para_list is None:
+        schema_para_list = ()
+
+    for param in schema_para_list:
+        if param in builtin_names:
+            continue
+
+        param_string_list.append(
+            """
+        <arg name="{param}">
+          <title>{param}</title>
+          <required_on_create>0</required_on_create>
+          <required_on_edit>0</required_on_edit>
+        </arg>
+        """.format(param=param)
+        )
+
+    description = ("Go to the add-on's configuration UI and configure"
+                   " modular inputs under the Inputs menu.")
+
+    print """
+    <scheme>
+    <title>{data_input_title}</title>
+    <description>{description}</description>
+    <use_external_validation>true</use_external_validation>
+    <streaming_mode>xml</streaming_mode>
+    <use_single_instance>{single_instance}</use_single_instance>
+    <endpoint>
+      <args>
+        <arg name="name">
+          <title>{data_input_title} Data Input Name</title>
+        </arg>
+        {param_str}
+      </args>
+    </endpoint>
+    </scheme>
+    """.format(
+        single_instance=(str(single_instance)).lower(),
+        data_input_title=mod_input_name,
+        param_str=''.join(param_string_list),
+        description=description,
+    )
+
+
+def _setup_signal_handler(data_loader, ta_short_name):
+    """
+    Setup signal handlers
+    :data_loader: data_loader.DataLoader instance
+    """
+
+    def _handle_exit(signum, frame):
+        stulog.logger.info("{} receives exit signal".format(ta_short_name))
+        if data_loader is not None:
+            data_loader.tear_down()
+
+    utils.handle_tear_down_signals(_handle_exit)
+
+
+def _handle_file_changes(data_loader):
+    """
+    :reload conf files and exit
+    """
+
+    def _handle_refresh(changed_files):
+        stulog.logger.info("Detect {} changed, reboot itself".format(
+            changed_files))
+        data_loader.tear_down()
+
+    return _handle_refresh
+
+
+def _get_conf_files(settings):
+    rest_root = settings.get("meta").get("restRoot")
+    file_list = [rest_root + "_settings.conf"]
+    if settings.get("pages") and settings.get("pages").get("configuration"):
+        configs = settings.get("pages").get("configuration")
+        tabs = configs.get("tabs") if configs.get("tabs") else []
+        for tab in tabs:
+            if tab.get("table"):
+                file_list.append(rest_root + "_" + tab.get("name") + ".conf")
+    ta_dir = get_app_root_dir()
+    return [op.join(ta_dir, "local", f) for f in file_list]
+
+
+def run(collector_cls, settings, checkpoint_cls=None, config_cls=None,
+        log_suffix=None, single_instance=True, cc_json_file=None):
+    """
+    Main loop. Run this TA forever
+    """
+    ta_short_name = settings["meta"]["name"].lower()
+
+    # This is for stdout flush
+    utils.disable_stdout_buffer()
+
+    # http://bugs.python.org/issue7980
+    time.strptime('2016-01-01', '%Y-%m-%d')
+
+    loader = dl.create_data_loader()
+
+    # handle signal
+    _setup_signal_handler(loader, ta_short_name)
+
+    # monitor files to reboot
+    try:
+        monitor = fm.FileMonitor(_handle_file_changes(loader),
+                                 _get_conf_files(settings))
+        loader.add_timer(monitor.check_changes, time.time(), 10)
+    except Exception:
+        stulog.logger.exception("Fail to add files for monitoring")
+
+    # add orphan process handling, which will check each 1 second
+    orphan_checker = opm.OrphanProcessChecker(loader.tear_down)
+    loader.add_timer(orphan_checker.check_orphan, time.time(), 1)
+
+    tconfig = tc.create_ta_config(settings, config_cls or tc.TaConfig,
+                                  log_suffix, single_instance=single_instance)
+    task_configs = tconfig.get_task_configs()
+
+    if not task_configs:
+        stulog.logger.debug("No task and exiting...")
+        return
+    meta_config = tconfig.get_meta_config()
+    meta_config["cc_json_file"] = cc_json_file
+
+    if tconfig.is_shc_member():
+        # Don't support SHC env
+        stulog.logger.error("This host is in search head cluster environment , "
+                            "will exit.")
+        return
+
+    # In this case, use file for checkpoint
+    if _is_checkpoint_dir_length_exceed_limit(tconfig,
+                                              meta_config["checkpoint_dir"]):
+        stulog.logger.error("The length of the checkpoint directory path: '{}' "
+                            "is too long. The max length we support is {}",
+                            meta_config["checkpoint_dir"],
+                            __CHECKPOINT_DIR_MAX_LEN__)
+        return
+
+    jobs = [
+        tdc.create_data_collector(
+            loader,
+            tconfig,
+            meta_config,
+            task_config,
+            collector_cls,
+            checkpoint_cls=checkpoint_cls or cpmgr.TACheckPointMgr
+        )
+        for task_config in task_configs
+        ]
+
+    loader.run(jobs)
+
+
+def _is_checkpoint_dir_length_exceed_limit(config, checkpoint_dir):
+    return platform.system() == 'Windows' \
+           and not config.is_search_head() \
+           and len(checkpoint_dir) >= __CHECKPOINT_DIR_MAX_LEN__
+
+
+def validate_config():
+    """
+    Validate inputs.conf
+    """
+
+    _, configs = modinput.get_modinput_configs_from_stdin()
+    return 0
+
+
+def usage():
+    """
+    Print usage of this binary
+    """
+
+    hlp = "%s --scheme|--validate-arguments|-h"
+    print >> sys.stderr, hlp % sys.argv[0]
+    sys.exit(1)
+
+
+def main(
+        collector_cls,
+        schema_file_path,
+        log_suffix="modinput",
+        checkpoint_cls=None,
+        config_cls=None,
+        cc_json_file=None,
+        schema_para_list=None,
+        single_instance=True
+):
+    """
+    Main entry point
+    """
+    assert collector_cls, "ucc modinput collector is None."
+    assert schema_file_path, "ucc modinput schema file is None"
+
+    settings = ld(schema_file_path)
+
+    mod_input_name = get_mod_input_script_name()
+
+    args = sys.argv
+    if len(args) > 1:
+        if args[1] == "--scheme":
+            do_scheme(
+                mod_input_name=mod_input_name,
+                schema_para_list=schema_para_list,
+                single_instance=single_instance
+            )
+        elif args[1] == "--validate-arguments":
+            sys.exit(validate_config())
+        elif args[1] in ("-h", "--h", "--help"):
+            usage()
+        else:
+            usage()
+    else:
+        try:
+            run(
+                collector_cls,
+                settings,
+                checkpoint_cls=checkpoint_cls,
+                config_cls=config_cls,
+                log_suffix=log_suffix,
+                single_instance=single_instance,
+                cc_json_file=cc_json_file
+            )
+        except Exception:
+            stulog.logger.exception(
+                "{} task encounter exception".format(mod_input_name))
+        stulog.logger.info("End {} task".format(mod_input_name))
+    sys.exit(0)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/splunk_ta_import_declare.py

@@ -0,0 +1,17 @@
+"""
+This module is used to filter and reload PATH.
+"""
+
+import os
+import sys
+import re
+
+ta_name = os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ta_lib_name = re.sub("[^\w]+", "_", ta_name.lower())
+assert ta_name or ta_name == "package", "TA name is None or package"
+pattern = re.compile(r"[\\/]etc[\\/]apps[\\/][^\\/]+[\\/]bin[\\/]?$")
+new_paths = [path for path in sys.path if not pattern.search(path) or ta_name in path]
+new_paths.insert(0, os.path.sep.join([os.path.dirname(__file__), ta_lib_name]))
+sys.path = new_paths
+
+

package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/ta_cloud_connect_client.py

@@ -0,0 +1,41 @@
+from .data_collection.ta_data_client import TaDataClient
+from ..splunktacollectorlib.common import log as stulog
+from ..splunktacollectorlib.data_collection import ta_consts as c
+from ..common.log import set_cc_logger
+
+
+class TACloudConnectClient(TaDataClient):
+    def __init__(self,
+                 meta_config,
+                 task_config,
+                 checkpoint_mgr=None,
+                 event_writer=None
+                 ):
+        super(TACloudConnectClient, self).__init__(meta_config,
+                                                   task_config,
+                                                   checkpoint_mgr,
+                                                   event_writer)
+        self._set_log()
+        self._cc_config_file = self._meta_config["cc_json_file"]
+        from ..core.pipemgr import PipeManager
+        from ..client import CloudConnectClient as Client
+        self._pipe_mgr = PipeManager(event_writer=event_writer)
+        self._client = Client(self._task_config, self._cc_config_file,
+                              checkpoint_mgr)
+
+    def _set_log(self):
+        pairs = ['{}="{}"'.format(c.stanza_name, self._task_config[
+            c.stanza_name])]
+        set_cc_logger(stulog.logger,
+                          logger_prefix="[{}]".format(" ".join(pairs)))
+
+    def is_stopped(self):
+        return self._stop
+
+    def stop(self):
+        self._stop = True
+        self._client.stop()
+
+    def get(self):
+        self._client.start()
+        raise StopIteration

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.9"
+__license__ = "Splunk"

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/consts.py

@@ -0,0 +1 @@
+util_log = "util"

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/log.py

@@ -0,0 +1,137 @@
+"""
+Copyright (C) 2005-2015 Splunk Inc. All Rights Reserved.
+
+log utility for TA
+"""
+
+import logging
+import logging.handlers as handlers
+import os
+import os.path as op
+
+from ..splunk_platform import make_splunkhome_path
+from . import util as cutil
+from .pattern import singleton
+import time
+logging.Formatter.converter = time.gmtime
+
+__LOG_FORMAT__ = "%(asctime)s +0000 log_level=%(levelname)s, pid=%(process)d, " \
+                 "tid=%(threadName)s, file=%(filename)s, " \
+                 "func_name=%(funcName)s, code_line_no=%(lineno)d | %(message)s"
+
+
+def log_enter_exit(logger):
+    """
+    Log decorator to log function enter and exit
+    """
+    def log_decorator(func):
+        def wrapper(*args, **kwargs):
+            logger.debug("{} entered.".format(func.__name__))
+            result = func(*args, **kwargs)
+            logger.debug("{} exited.".format(func.__name__))
+            return result
+        return wrapper
+    return log_decorator
+
+
+def check_add_stderr_handler():
+    env_var = os.environ.get('splunk.cloudconnect.settings.logging.type')
+    return env_var and env_var == "stderr"
+
+@singleton
+class Logs(object):
+
+    def __init__(self, namespace=None, default_level=logging.INFO):
+        self._loggers = {}
+        self._default_level = default_level
+        if namespace is None:
+            namespace = cutil.get_appname_from_path(op.abspath(__file__))
+
+        if namespace:
+            namespace = namespace.lower()
+        self._namespace = namespace
+
+    def get_logger(self, name, level=None,
+                   maxBytes=25000000, backupCount=5):
+        """
+        Set up a default logger.
+
+        :param name: The log file name.
+        :param level: The logging level.
+        :param maxBytes: The maximum log file size before rollover.
+        :param backupCount: The number of log files to retain.
+        """
+
+        # Strip ".py" from the log file name if auto-generated by a script.
+        if level is None:
+            level = self._default_level
+
+        name = self._get_log_name(name)
+        if name in self._loggers:
+            return self._loggers[name]
+
+        logger = logging.getLogger(name)
+
+        if check_add_stderr_handler():
+            import sys
+            ch = logging.StreamHandler(sys.stderr)
+            ch.setLevel(logging.ERROR)
+            formatter = logging.Formatter(__LOG_FORMAT__)
+            ch.setFormatter(formatter)
+            logger.addHandler(ch)
+        else:
+            logfile = make_splunkhome_path(["var", "log", "splunk", name])
+            handler_exists = any(
+                [True for h in logger.handlers if h.baseFilename == logfile])
+            if not handler_exists:
+                file_handler = handlers.RotatingFileHandler(
+                    logfile, mode="a", maxBytes=maxBytes, backupCount=backupCount)
+                formatter = logging.Formatter(__LOG_FORMAT__ )
+                file_handler.setFormatter(formatter)
+                logger.addHandler(file_handler)
+                logger.setLevel(level)
+                logger.propagate = False
+
+        self._loggers[name] = logger
+        return logger
+
+    def set_level(self, level, name=None):
+        """
+        Change the log level of the logging
+
+        :param level: the level of the logging to be setLevel
+        :param name: the name of the logging to set, in case it is not set,
+                     all the loggers will be affected
+        """
+
+        if name is not None:
+            name = self._get_log_name(name)
+            logger = self._loggers.get(name)
+            if logger is not None:
+                logger.setLevel(level)
+        else:
+            self._default_level = level
+            for logger in self._loggers.itervalues():
+                logger.setLevel(level)
+
+    def _get_log_name(self, name):
+        if name.endswith(".py"):
+            name = name.replace(".py", "")
+
+        if self._namespace:
+            name = "{}_{}.log".format(self._namespace, name)
+        else:
+            name = "{}.log" .format(name)
+        return name
+
+# Global logger
+logger = Logs().get_logger("util")
+
+
+def reset_logger(name):
+    """
+    Reset global logger.
+    """
+
+    global logger
+    logger = Logs().get_logger(name)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/pattern.py

@@ -0,0 +1,37 @@
+"""
+Copyright (C) 2005-2015 Splunk Inc. All Rights Reserved.
+
+Commonly used design partten for python user, includes:
+  - singleton (Decorator function used to build singleton)
+"""
+
+from functools import wraps
+
+
+def singleton(class_):
+    """
+    Singleton decoorator function.
+    """
+    instances = {}
+
+    @wraps(class_)
+    def getinstance(*args, **kwargs):
+        if class_ not in instances:
+            instances[class_] = class_(*args, **kwargs)
+        return instances[class_]
+    return getinstance
+
+
+class Singleton(type):
+    """
+    Singleton meta class
+    """
+
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(
+                *args, **kwargs)
+            print cls
+        return cls._instances[cls]

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/util.py

@@ -0,0 +1,108 @@
+"""
+Copyright (C) 2005-2015 Splunk Inc. All Rights Reserved.
+"""
+
+import os
+import os.path as op
+import datetime
+import sys
+import gc
+import urllib
+
+
+def handle_tear_down_signals(callback):
+    import signal
+
+    signal.signal(signal.SIGTERM, callback)
+    signal.signal(signal.SIGINT, callback)
+
+    if os.name == "nt":
+        signal.signal(signal.SIGBREAK, callback)
+
+
+def datetime_to_seconds(dt):
+    epoch_time = datetime.datetime.utcfromtimestamp(0)
+    return (dt - epoch_time).total_seconds()
+
+
+def is_true(val):
+    value = str(val).strip().upper()
+    if value in ("1", "TRUE", "T", "Y", "YES"):
+        return True
+    return False
+
+
+def is_false(val):
+    value = str(val).strip().upper()
+    if value in ("0", "FALSE", "F", "N", "NO", "NONE", ""):
+        return True
+    return False
+
+
+def remove_http_proxy_env_vars():
+    for k in ("http_proxy", "https_proxy"):
+        if k in os.environ:
+            del os.environ[k]
+        elif k.upper() in os.environ:
+            del os.environ[k.upper()]
+
+
+def get_appname_from_path(absolute_path):
+    absolute_path = op.normpath(absolute_path)
+    parts = absolute_path.split(os.path.sep)
+    parts.reverse()
+    for key in ("apps", "slave-apps", "master-apps"):
+        try:
+            idx = parts.index(key)
+        except ValueError:
+            continue
+        else:
+            try:
+                if parts[idx + 1] == "etc":
+                    return parts[idx - 1]
+            except IndexError:
+                pass
+            continue
+    #return None
+    return "-"
+
+
+def escape_cdata(data):
+    # FIXME: This is a workaround for JIRA [addon-10459]
+    data = data.decode("utf-8", errors="replace").encode("utf-8", errors="xmlcharrefreplace")
+    data = data.replace("]]>", "]]>")
+    if data.endswith("]"):
+        data = data[:-1] + "%5D"
+    return data
+
+
+def extract_datainput_name(stanza_name):
+    """
+    stansa_name: string like aws_s3://my_s3_data_input
+    """
+
+    sep = "://"
+    try:
+        idx = stanza_name.index(sep)
+    except ValueError:
+        return stanza_name
+
+    return stanza_name[idx + len(sep):]
+
+
+def escape_json_control_chars(json_str):
+    control_chars = ((r"\n", "\\\\n"), (r"\r", "\\\\r"),
+                     (r"\r\n", "\\\\r\\\\n"))
+    for ch, replace in control_chars:
+        json_str = json_str.replace(ch, replace)
+    return json_str
+
+
+def disable_stdout_buffer():
+    os.environ["PYTHONUNBUFFERED"] = "1"
+    sys.stdout = os.fdopen(sys.stdout.fileno(), "w", 0)
+    gc.garbage.append(sys.stdout)
+
+
+def format_stanza_name(name):
+    return urllib.quote(name.encode("utf-8"), "")

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/xml_dom_parser.py

@@ -0,0 +1,47 @@
+import re
+from xml.etree import cElementTree as et
+
+
+def parse_conf_xml_dom(xml_content):
+    """
+    @xml_content: XML DOM from splunkd
+    """
+
+    m = re.search(r'xmlns="([^"]+)"', xml_content)
+    ns = m.group(1)
+    m = re.search(r'xmlns:s="([^"]+)"', xml_content)
+    sub_ns = m.group(1)
+    entry_path = "./{%s}entry" % ns
+    stanza_path = "./{%s}title" % ns
+    key_path = "./{%s}content/{%s}dict/{%s}key" % (ns, sub_ns, sub_ns)
+    meta_path = "./{%s}dict/{%s}key" % (sub_ns, sub_ns)
+    list_path = "./{%s}list/{%s}item" % (sub_ns, sub_ns)
+
+    xml_conf = et.fromstring(xml_content)
+    stanza_objs = []
+    for entry in xml_conf.iterfind(entry_path):
+        for stanza in entry.iterfind(stanza_path):
+            stanza_obj = {"name": stanza.text,"stanza": stanza.text}
+            break
+        else:
+            continue
+
+        for key in entry.iterfind(key_path):
+            if key.get("name") == "eai:acl":
+                meta = {}
+                for k in key.iterfind(meta_path):
+                    meta[k.get("name")] = k.text
+                stanza_obj[key.get("name")] = meta
+            elif key.get("name") != "eai:attributes":
+                name = key.get("name")
+                if name.startswith("eai:"):
+                    name = name[4:]
+                list_vals = [k.text for k in key.iterfind(list_path)]
+                if list_vals:
+                    stanza_obj[name] = list_vals
+                else:
+                    stanza_obj[name] = key.text
+                    if key.text == "None":
+                        stanza_obj[name] = None
+        stanza_objs.append(stanza_obj)
+    return stanza_objs

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/concurrent/concurrent_executor.py

@@ -0,0 +1,88 @@
+"""
+Concurrent executor provides concurrent executing function either in
+a thread pool or a process pool
+"""
+
+from ..concurrent import thread_pool as tp
+from ..concurrent import process_pool as pp
+
+
+class ConcurrentExecutor(object):
+
+    def __init__(self, config):
+        """
+        :param config: dict like object, contains thread_min_size (int),
+                       thread_max_size (int), daemonize_thread (bool),
+                       process_size (int)
+        """
+
+        self._io_executor = tp.ThreadPool(config.get("thread_min_size", 0),
+                                          config.get("thread_max_size", 0),
+                                          config.get("task_queue_size", 1024),
+                                          config.get("daemonize_thread", True))
+        self._compute_executor = None
+        if config.get("process_size", 0):
+            self._compute_executor = pp.ProcessPool(
+                config.get("process_size", 0))
+
+    def start(self):
+        self._io_executor.start()
+
+    def tear_down(self):
+        self._io_executor.tear_down()
+        if self._compute_executor is not None:
+            self._compute_executor.tear_down()
+
+    def run_io_func_sync(self, func, args=(), kwargs=None):
+        """
+        :param func: callable
+        :param args: free params
+        :param kwargs: named params
+        :return whatever the func returns
+        """
+
+        return self._io_executor.apply(func, args, kwargs)
+
+    def run_io_func_async(self, func, args=(), kwargs=None, callback=None):
+        """
+        :param func: callable
+        :param args: free params
+        :param kwargs: named params
+        :calllback: when func is done and without exception, call the callback
+        :return whatever the func returns
+        """
+
+        return self._io_executor.apply_async(func, args, kwargs, callback)
+
+    def enqueue_io_funcs(self, funcs, block=True):
+        """
+        run jobs in a fire and forget way, no result will be handled
+        over to clients
+        :param funcs: tuple/list-like or generator like object, func shall be
+                      callable
+        """
+
+        return self._io_executor.enqueue_funcs(funcs, block)
+
+    def run_compute_func_sync(self, func, args=(), kwargs={}):
+        """
+        :param func: callable
+        :param args: free params
+        :param kwargs: named params
+        :return whatever the func returns
+        """
+
+        assert self._compute_executor is not None
+        return self._compute_executor.apply(func, args, kwargs)
+
+    def run_compute_func_async(self, func, args=(), kwargs={}, callback=None):
+        """
+        :param func: callable
+        :param args: free params
+        :param kwargs: named params
+        :calllback: when func is done and without exception, call the callback
+        :return whatever the func returns
+        """
+
+        assert self._compute_executor is not None
+        return self._compute_executor.apply_async(func, args, kwargs, callback)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/concurrent/process_pool.py

@@ -0,0 +1,63 @@
+"""
+A wrapper of multiprocessing.pool
+"""
+
+import multiprocessing
+from ..common import log
+
+
+class ProcessPool(object):
+    """
+    A simple wrapper of multiprocessing.pool
+    """
+
+    def __init__(self, size=0, maxtasksperchild=10000):
+        if size <= 0:
+            size = multiprocessing.cpu_count()
+        self.size = size
+        self._pool = multiprocessing.Pool(processes=size,
+                                          maxtasksperchild=maxtasksperchild)
+        self._stopped = False
+
+    def tear_down(self):
+        """
+        Tear down the pool
+        """
+
+        if self._stopped:
+            log.logger.info("ProcessPool has already stopped.")
+            return
+        self._stopped = True
+
+        self._pool.close()
+        self._pool.join()
+        log.logger.info("ProcessPool stopped.")
+
+    def apply(self, func, args=(), kwargs={}):
+        """
+        :param func: callable
+        :param args: free params
+        :param kwargs: named params
+        :return whatever the func returns
+        """
+
+        if self._stopped:
+            log.logger.info("ProcessPool has already stopped.")
+            return None
+
+        return self._pool.apply(func, args, kwargs)
+
+    def apply_async(self, func, args=(), kwargs={}, callback=None):
+        """
+        :param func: callable
+        :param args: free params
+        :param kwargs: named params
+        :callback: when func is done without exception, call this callack
+        :return whatever the func returns
+        """
+
+        if self._stopped:
+            log.logger.info("ProcessPool has already stopped.")
+            return None
+
+        return self._pool.apply_async(func, args, kwargs, callback)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/concurrent/thread_pool.py

@@ -0,0 +1,344 @@
+"""
+A simple thread pool implementation
+"""
+
+import threading
+import Queue
+import multiprocessing
+import traceback
+import exceptions
+from time import time
+
+from ..common import log
+
+
+class ThreadPool(object):
+    """
+    A simple thread pool implementation
+    """
+
+    _high_watermark = 0.2
+    _resize_window = 10
+
+    def __init__(self, min_size=1, max_size=128,
+                 task_queue_size=1024, daemon=True):
+        assert task_queue_size
+
+        if not min_size or min_size <= 0:
+            min_size = multiprocessing.cpu_count()
+
+        if not max_size or max_size <= 0:
+            max_size = multiprocessing.cpu_count() * 8
+
+        self._min_size = min_size
+        self._max_size = max_size
+        self._daemon = daemon
+
+        self._work_queue = Queue.Queue(task_queue_size)
+        self._thrs = []
+        for _ in range(min_size):
+            thr = threading.Thread(target=self._run)
+            self._thrs.append(thr)
+        self._admin_queue = Queue.Queue()
+        self._admin_thr = threading.Thread(target=self._do_admin)
+        self._last_resize_time = time()
+        self._last_size = min_size
+        self._lock = threading.Lock()
+        self._occupied_threads = 0
+        self._count_lock = threading.Lock()
+        self._started = False
+
+    def start(self):
+        """
+        Start threads in the pool
+        """
+
+        with self._lock:
+            if self._started:
+                return
+            self._started = True
+
+            for thr in self._thrs:
+                thr.daemon = self._daemon
+                thr.start()
+
+            self._admin_thr.start()
+        log.logger.info("ThreadPool started.")
+
+    def tear_down(self):
+        """
+        Tear down thread pool
+        """
+
+        with self._lock:
+            if not self._started:
+                return
+            self._started = False
+
+            for thr in self._thrs:
+                self._work_queue.put(None, block=True)
+
+            self._admin_queue.put(None)
+
+            if not self._daemon:
+                log.logger.info("Wait for threads to stop.")
+                for thr in self._thrs:
+                    thr.join()
+            self._admin_thr.join()
+
+        log.logger.info("ThreadPool stopped.")
+
+    def enqueue_funcs(self, funcs, block=True):
+        """
+        run jobs in a fire and forget way, no result will be handled
+        over to clients
+        :param funcs: tuple/list-like or generator like object, func shall be
+                      callable
+        """
+
+        if not self._started:
+            log.logger.info("ThreadPool has already stopped.")
+            return
+
+        for func in funcs:
+            self._work_queue.put(func, block)
+
+    def apply_async(self, func, args=(), kwargs=None, callback=None):
+        """
+        :param func: callable
+        :param args: free params
+        :param kwargs: named params
+        :callback: when func is done and without exception, call the callback
+        :return AsyncResult, clients can poll or wait the result through it
+        """
+
+        if not self._started:
+            log.logger.info("ThreadPool has already stopped.")
+            return None
+
+        res = AsyncResult(func, args, kwargs, callback)
+        self._work_queue.put(res)
+        return res
+
+    def apply(self, func, args=(), kwargs=None):
+        """
+        :param func: callable
+        :param args: free params
+        :param kwargs: named params
+        :return whatever the func returns
+        """
+
+        if not self._started:
+            log.logger.info("ThreadPool has already stopped.")
+            return None
+
+        res = self.apply_async(func, args, kwargs)
+        return res.get()
+
+    def size(self):
+        return self._last_size
+
+    def resize(self, new_size):
+        """
+        Resize the pool size, spawn or destroy threads if necessary
+        """
+
+        if new_size <= 0:
+            return
+
+        if self._lock.locked() or not self._started:
+            log.logger.info("Try to resize thread pool during the tear "
+                            "down process, do nothing")
+            return
+
+        with self._lock:
+            self._remove_exited_threads_with_lock()
+            size = self._last_size
+            self._last_size = new_size
+            if new_size > size:
+                for _ in xrange(new_size - size):
+                    thr = threading.Thread(target=self._run)
+                    thr.daemon = self._daemon
+                    thr.start()
+                    self._thrs.append(thr)
+            elif new_size < size:
+                for _ in xrange(size - new_size):
+                    self._work_queue.put(None)
+        log.logger.info("Finished ThreadPool resizing. New size=%d", new_size)
+
+    def _remove_exited_threads_with_lock(self):
+        """
+        Join the exited threads last time when resize was called
+        """
+
+        joined_thrs = set()
+        for thr in self._thrs:
+            if not thr.is_alive():
+                try:
+                    if not thr.daemon:
+                        thr.join(timeout=0.5)
+                    joined_thrs.add(thr.ident)
+                except RuntimeError:
+                    pass
+
+        if joined_thrs:
+            live_thrs = []
+            for thr in self._thrs:
+                if thr.ident not in joined_thrs:
+                    live_thrs.append(thr)
+            self._thrs = live_thrs
+
+    def _do_resize_according_to_loads(self):
+        if (self._last_resize_time and
+                time() - self._last_resize_time < self._resize_window):
+            return
+
+        thr_size = self._last_size
+        free_thrs = thr_size - self._occupied_threads
+        work_size = self._work_queue.qsize()
+
+        log.logger.debug("current_thr_size=%s, free_thrs=%s, work_size=%s",
+                        thr_size, free_thrs, work_size)
+        if work_size and work_size > free_thrs:
+            if thr_size < self._max_size:
+                thr_size = min(thr_size * 2, self._max_size)
+                self.resize(thr_size)
+        elif free_thrs > 0:
+            free = free_thrs * 1.0
+            if free / thr_size >= self._high_watermark and free_thrs >= 2:
+                # 20 % thrs are idle, tear down half of the idle ones
+                thr_size = thr_size - free_thrs / 2
+                if thr_size > self._min_size:
+                    self.resize(thr_size)
+        self._last_resize_time = time()
+
+    def _do_admin(self):
+        admin_q = self._admin_queue
+        resize_win = self._resize_window
+        while 1:
+            try:
+                wakup = admin_q.get(timeout=resize_win + 1)
+            except Queue.Empty:
+                self._do_resize_according_to_loads()
+                continue
+
+            if wakup is None:
+                break
+            else:
+                self._do_resize_according_to_loads()
+        log.logger.info("ThreadPool admin thread=%s stopped.",
+                        threading.current_thread().getName())
+
+    def _run(self):
+        """
+        Threads callback func, run forever to handle jobs from the job queue
+        """
+
+        work_queue = self._work_queue
+        count_lock = self._count_lock
+        while 1:
+            log.logger.debug("Going to get job")
+            func = work_queue.get()
+            if func is None:
+                break
+
+            if not self._started:
+                break
+
+            log.logger.debug("Going to exec job")
+            with count_lock:
+                self._occupied_threads += 1
+
+            try:
+                func()
+            except Exception:
+                log.logger.error(traceback.format_exc())
+
+            with count_lock:
+                self._occupied_threads -= 1
+
+            log.logger.debug("Done with exec job")
+            log.logger.info("Thread work_queue_size=%d", work_queue.qsize())
+
+        log.logger.debug("Worker thread %s stopped.",
+                         threading.current_thread().getName())
+
+
+class AsyncResult(object):
+
+    def __init__(self, func, args, kwargs, callback):
+        self._func = func
+        self._args = args
+        self._kwargs = kwargs
+        self._callback = callback
+        self._q = Queue.Queue()
+
+    def __call__(self):
+        try:
+            if self._args and self._kwargs:
+                res = self._func(*self._args, **self._kwargs)
+            elif self._args:
+                res = self._func(*self._args)
+            elif self._kwargs:
+                res = self._func(**self._kwargs)
+            else:
+                res = self._func()
+        except Exception as e:
+            self._q.put(e)
+            return
+        else:
+            self._q.put(res)
+
+        if self._callback is not None:
+            self._callback()
+
+    def get(self, timeout=None):
+        """
+        Return the result when it arrives. If timeout is not None and the
+        result does not arrive within timeout seconds then
+        multiprocessing.TimeoutError is raised. If the remote call raised an
+        exception then that exception will be reraised by get().
+        """
+
+        try:
+            res = self._q.get(timeout=timeout)
+        except Queue.Empty:
+            raise multiprocessing.TimeoutError("Timed out")
+
+        if isinstance(res, Exception):
+            raise res
+        return res
+
+    def wait(self, timeout=None):
+        """
+        Wait until the result is available or until timeout seconds pass.
+        """
+
+        try:
+            res = self._q.get(timeout=timeout)
+        except Queue.Empty:
+            pass
+        else:
+            self._q.put(res)
+
+    def ready(self):
+        """
+        Return whether the call has completed.
+        """
+
+        return len(self._q)
+
+    def successful(self):
+        """
+        Return whether the call completed without raising an exception.
+        Will raise AssertionError if the result is not ready.
+        """
+
+        if not self.ready():
+            raise exceptions.AssertionError("Function is not ready")
+        res = self._q.get()
+        self._q.put(res)
+
+        if isinstance(res, Exception):
+            return False
+        return True

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/conf_endpoints.py

@@ -0,0 +1,142 @@
+from .request import content_request
+from ..common import util
+from ..common import xml_dom_parser as xdp
+
+CONF_ENDPOINT = "%s/servicesNS/%s/%s/configs/conf-%s"
+
+
+def _conf_endpoint_ns(uri, owner, app, conf_name):
+    return CONF_ENDPOINT % (uri, owner, app, conf_name)
+
+
+def reload_conf(splunkd_uri, session_key, app_name, conf_name, throw=False):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param conf_names: a list of the name of the conf file, e.g. ["props"]
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    """
+
+    uri = _conf_endpoint_ns(splunkd_uri, "nobody", app_name, conf_name)
+    uri += "/_reload"
+    msg = "Failed to reload conf in app=%s: %s" % (app_name, conf_name)
+
+    try:
+        content_request(uri, session_key, "GET", None, msg)
+    except Exception:
+        if throw:
+            raise
+
+
+def create_stanza(splunkd_uri, session_key, owner, app_name, conf_name,
+                  stanza, key_values):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param conf_name: the name of the conf file, e.g. "props"
+    :param stanza: stanza name, e.g. "aws:cloudtrail"
+    :param key_values: the key-value dict of the stanza
+    :return: None on success otherwise throw exception
+    """
+
+    uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
+    msg = "Failed to create stanza=%s in conf=%s" % (stanza, conf_name)
+    payload = {"name": unicode(stanza).encode('utf-8')}
+    for key in key_values:
+        if key != "name":
+            payload[key] = str(key_values[key])
+
+    content_request(uri, session_key, "POST", payload, msg)
+
+
+def get_conf(splunkd_uri, session_key, owner, app_name, conf_name,
+             stanza=None):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param conf_name: the name of the conf file, e.g. "props"
+    :param stanza: stanza name, e.g. "aws:cloudtrail"
+    :return: a list of stanzas in the conf file, including metadata
+    """
+
+    uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
+
+    if stanza:
+        uri += "/" + util.format_stanza_name(stanza)
+
+    # get all the stanzas at one time
+    uri += "?count=0&offset=0"
+
+    msg = "Failed to get stanza=%s in conf=%s" % (stanza if stanza else stanza, conf_name)
+    content = content_request(uri, session_key, "GET", None, msg)
+    return xdp.parse_conf_xml_dom(content)
+
+
+def update_stanza(splunkd_uri, session_key, owner, app_name, conf_name,
+                  stanza, key_values):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param conf_name: the name of the conf file, e.g. "props"
+    :param stanza: stanza name, e.g. "aws:cloudtrail"
+    :param key_values: the key-value dict of the stanza
+    :return: None on success otherwise raise exception
+    """
+
+    uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
+    uri += "/" + util.format_stanza_name(stanza)
+    msg = "Failed to update stanza=%s in conf=%s" % (stanza, conf_name)
+    return content_request(uri, session_key, "POST", key_values, msg)
+
+
+def delete_stanza(splunkd_uri, session_key, owner, app_name, conf_name,
+                  stanza, throw=False):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param conf_name: the name of the conf file, e.g. "props"
+    :param stanza: stanza name, e.g. "aws:cloudtrail"
+    :return: None on success otherwise raise exception
+    """
+
+    uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
+    uri += "/" + util.format_stanza_name(stanza)
+    msg = "Failed to delete stanza=%s in conf=%s" % (stanza, conf_name)
+    content_request(uri, session_key, "DELETE", None, msg)
+
+
+def stanza_exist(splunkd_uri, session_key, owner, app_name, conf_name,
+                 stanza):
+    try:
+        res = get_conf(splunkd_uri, session_key, owner, app_name, conf_name,
+                       stanza)
+        return len(res) > 0
+    except Exception:
+        return False
+
+
+def operate_conf(splunkd_uri, session_key, owner, app_name, conf_name,
+                 stanza, operation):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param conf_name: the name of the conf file, e.g. "props"
+    :param stanza: stanza name, e.g. "aws:cloudtrail"
+    :param operation: must be "disable" or "enable"
+    """
+
+    assert operation in ("disable", "enable")
+    uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
+    uri += "/%s/%s" % (util.format_stanza_name(stanza), operation)
+    msg = "Failed to disable/enable stanza=%s in conf=%s" % (stanza, conf_name)
+    content_request(uri, session_key, "POST", None, msg)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/conf_manager.py

@@ -0,0 +1,225 @@
+"""
+This module hanles configuration related stuff
+"""
+
+import os.path as op
+
+from . import conf_endpoints as scmc
+from . import data_input_endpoints as scmdi
+from . import property_endpoints as scmp
+from . import request as req
+
+
+def conf_file2name(conf_file):
+    conf_name = op.basename(conf_file)
+    if conf_name.endswith(".conf"):
+        conf_name = conf_name[:-5]
+    return conf_name
+
+
+class ConfManager(object):
+
+    def __init__(self, splunkd_uri, session_key, owner="nobody", app_name="-"):
+        """
+        :app_name: when creating conf stanza, app_name is required to set not
+        to "-"
+        :owner: when creating conf stanza, app_name is required to set not
+        to "-"
+        """
+
+        self.splunkd_uri = splunkd_uri
+        self.session_key = session_key
+        self.owner = owner
+        self.app_name = app_name
+
+    def set_appname(self, appname):
+        """
+        This are cases we need edit/remove/create confs in different app
+        context. call this interface to switch app context before manipulate
+        the confs in different app context
+        """
+
+        self.app_name = appname
+
+    def all_stanzas(self, conf_name, do_reload=False, ret_metadata=False):
+        """
+        :return: a list of dict stanza objects if successful.
+                 Otherwise raise exception
+        """
+
+        if do_reload:
+            self.reload_conf(conf_name)
+
+        stanzas = scmc.get_conf(self.splunkd_uri, self.session_key,
+                                "-", "-", conf_name)
+        return self._delete_metadata(stanzas, ret_metadata)
+
+    def all_stanzas_as_dicts(self, conf_name, do_reload=False,
+                             ret_metadata=False):
+        """
+        :return: a dict of dict stanza objects if successful.
+                 otherwise raise exception
+        """
+
+        stanzas = self.all_stanzas(conf_name, do_reload, ret_metadata)
+        return {stanza["name"]: stanza for stanza in stanzas}
+
+    def get_stanza(self, conf_name, stanza,
+                   do_reload=False, ret_metadata=False):
+        """
+        @return dict if success otherwise raise exception
+        """
+
+        if do_reload:
+            self.reload_conf(conf_name)
+
+        stanzas = scmc.get_conf(self.splunkd_uri, self.session_key,
+                                "-", "-", conf_name, stanza)
+        stanzas = self._delete_metadata(stanzas, ret_metadata)
+        return stanzas[0]
+
+    def reload_conf(self, conf_name):
+        scmc.reload_conf(self.splunkd_uri, self.session_key, "-", conf_name)
+
+    def enable_conf(self, conf_name, stanza):
+        scmc.operate_conf(self.splunkd_uri, self.session_key,
+                          self.owner, self.app_name,
+                          conf_name, stanza, "enable")
+
+    def disable_conf(self, conf_name, stanza):
+        scmc.operate_conf(self.splunkd_uri, self.session_key,
+                          self.owner, self.app_name,
+                          conf_name, stanza, "disable")
+
+    def get_property(self, conf_name, stanza, key, do_reload=False):
+        if do_reload:
+            self.reload_conf(conf_name)
+
+        return scmp.get_property(self.splunkd_uri, self.session_key,
+                                 "-", "-", conf_name, stanza, key)
+
+    def stanza_exist(self, conf_name, stanza):
+        return scmc.stanza_exist(self.splunkd_uri, self.session_key,
+                                 "-", "-", conf_name, stanza)
+
+    def create_stanza(self, conf_name, stanza, key_values):
+        scmc.create_stanza(self.splunkd_uri, self.session_key,
+                           self.owner, self.app_name,
+                           conf_name, stanza, key_values)
+
+    def update_stanza(self, conf_name, stanza, key_values):
+        scmc.update_stanza(self.splunkd_uri, self.session_key,
+                           self.owner, self.app_name,
+                           conf_name, stanza, key_values)
+
+    def delete_stanza(self, conf_name, stanza):
+        scmc.delete_stanza(self.splunkd_uri, self.session_key,
+                           self.owner, self.app_name,
+                           conf_name, stanza)
+
+    def create_properties(self, conf_name, stanza):
+        scmp.create_properties(self.splunkd_uri, self.session_key,
+                               self.owner, self.app_name,
+                               conf_name, stanza)
+
+    def update_properties(self, conf_name, stanza, key_values):
+        scmp.update_properties(self.splunkd_uri, self.session_key,
+                               self.owner, self.app_name,
+                               conf_name, stanza, key_values)
+
+    def delete_stanzas(self, conf_name, stanzas):
+        """
+        :param stanzas: list of stanzas
+        :return: list of failed stanzas
+        """
+
+        failed_stanzas = []
+        for stanza in stanzas:
+            try:
+                self.delete_stanza(conf_name, stanza)
+            except Exception:
+                failed_stanzas.append(stanza)
+        return failed_stanzas
+
+    # data input management
+    def create_data_input(self, input_type, name, key_values=None):
+        scmdi.create_data_input(self.splunkd_uri, self.session_key,
+                                self.owner, self.app_name,
+                                input_type, name, key_values)
+
+    def update_data_input(self, input_type, name, key_values):
+        scmdi.update_data_input(self.splunkd_uri, self.session_key,
+                                self.owner, self.app_name,
+                                input_type, name, key_values)
+
+    def delete_data_input(self, input_type, name):
+        scmdi.delete_data_input(self.splunkd_uri, self.session_key,
+                                self.owner, self.app_name,
+                                input_type, name)
+
+    def get_data_input(self, input_type, name=None, do_reload=False):
+        if do_reload:
+            self.reload_data_input(input_type)
+
+        return scmdi.get_data_input(self.splunkd_uri, self.session_key,
+                                    "-", "-", input_type, name)
+
+    def reload_data_input(self, input_type):
+        scmdi.reload_data_input(self.splunkd_uri, self.session_key,
+                                "-", "-", input_type)
+
+    def enable_data_input(self, input_type, name):
+        scmdi.operate_data_input(self.splunkd_uri, self.session_key,
+                                 self.owner, self.app_name,
+                                 input_type, name, "enable")
+
+    def disable_data_input(self, input_type, name):
+        scmdi.operate_data_input(self.splunkd_uri, self.session_key,
+                                 self.owner, self.app_name,
+                                 input_type, name, "disable")
+
+    def data_input_exist(self, input_type, name):
+        try:
+            result = self.get_data_input(input_type, name)
+        except req.ConfNotExistsException:
+            return False
+
+        return result is not None
+
+    def all_data_input_stanzas(self, input_type, do_reload=False,
+                               ret_metadata=False):
+        stanzas = self.get_data_input(input_type, do_reload=do_reload)
+        for stanza in stanzas:
+            if "eai:acl" in stanza and "app" in stanza["eai:acl"]:
+                stanza["appName"] = stanza["eai:acl"]["app"]
+                stanza["userName"] = stanza["eai:acl"].get("owner", "nobody")
+        return self._delete_metadata(stanzas, ret_metadata)
+
+    def get_data_input_stanza(self, input_type, name, do_reload=False,
+                              ret_metadata=False):
+
+        stanzas = self.get_data_input(input_type, name, do_reload)
+        stanzas = self._delete_metadata(stanzas, ret_metadata)
+        return stanzas[0]
+
+    def delete_data_input_stanzas(self, input_type, names):
+        """
+        :param stanzas: list of stanzas
+        :return: list of failed stanzas
+        """
+
+        failed_names = []
+        for name in names:
+            try:
+                self.delete_data_input(input_type, name)
+            except Exception:
+                failed_names.append(name)
+        return failed_names
+
+    def _delete_metadata(self, stanzas, ret_metadata):
+        if stanzas and not ret_metadata:
+            for stanza in stanzas:
+                for key in stanza.keys():
+                    if key.startswith("eai:"):
+                        del stanza[key]
+        return stanzas

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/data_input_endpoints.py

@@ -0,0 +1,152 @@
+from .request import content_request
+from ..common import util
+from ..common import xml_dom_parser as xdp
+
+INPUT_ENDPOINT = "%s/servicesNS/%s/%s/data/inputs/%s"
+
+
+def _input_endpoint_ns(uri, owner, app, input_type):
+    return INPUT_ENDPOINT % (uri, owner, app, input_type)
+
+
+def reload_data_input(splunkd_uri, session_key, owner, app_name,
+                      input_type, throw=False):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param input_type: name of the input type.
+                       if it is a script input, the input is "script",
+                       for modinput, say snow, the input is "snow"
+    """
+
+    uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
+    uri += "/_reload"
+    msg = "Failed to reload data input in app=%s: %s" % (app_name, input_type)
+    try:
+        content_request(uri, session_key, "GET", None, msg)
+    except Exception:
+        if throw:
+            raise
+
+
+def create_data_input(splunkd_uri, session_key, owner, app_name, input_type,
+                      name, key_values):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param input_type: name of the input type.
+                       if it is a script input, the input is "script",
+                       for modinput, say snow, the input is "snow"
+    :param name: The name of the input stanza to create.
+                 i.e. stanza [<input_type>://<name>] will be created.
+    :param key_values: a K-V dict of details in the data input stanza.
+    :return: None on success else raise exception
+    """
+
+    key_values["name"] = unicode(name).encode('utf-8')
+    uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
+    msg = "Failed to create data input in app=%s: %s://%s" % (
+        app_name, input_type, name)
+    content_request(uri, session_key, "POST", key_values, msg)
+
+
+def get_data_input(splunkd_uri, session_key, owner, app_name, input_type,
+                   name=None):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param input_type: name of the input type.
+                       if it is a script input, the input is "script",
+                       for modinput, say snow, the input is "snow"
+    :param name: The name of the input stanza to create.
+                 i.e. stanza [<input_type>://<name>] will be deleted.
+    :return: a list of stanzas in the input type, including metadata
+    """
+
+    uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
+    if name:
+        uri += "/" + util.format_stanza_name(name)
+
+    # get all the stanzas at one time
+    uri += "?count=0&offset=0"
+
+    msg = "Failed to get data input in app=%s: %s://%s" % (
+        app_name, input_type, name if name else name)
+    content = content_request(uri, session_key, "GET", None, msg)
+    return xdp.parse_conf_xml_dom(content)
+
+
+def update_data_input(splunkd_uri, session_key, owner, app_name, input_type,
+                      name, key_values):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param input_type: name of the input type.
+                       if it is a script input, the input is "script",
+                       for modinput, say snow, the input is "snow"
+    :param name: The name of the input stanza to create.
+                 i.e. stanza [<input_type>://<name>] will be updated.
+    :param key_values: a K-V dict of details in the data input stanza.
+    :return: raise exception when failure
+    """
+
+    if "name" in key_values:
+        del key_values["name"]
+    uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
+    uri += "/" + util.format_stanza_name(name)
+    msg = "Failed to update data input in app=%s: %s://%s" % (
+        app_name, input_type, name)
+    content_request(uri, session_key, "POST", key_values, msg)
+
+
+def delete_data_input(splunkd_uri, session_key, owner, app_name, input_type,
+                      name):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param input_type: name of the input type.
+                       if it is a script input, the input is "script",
+                       for modinput, say snow, the input is "snow"
+    :param name: The name of the input stanza to create.
+                 i.e. stanza [<input_type>://<name>] will be deleted.
+    :return raise exception when failed
+    """
+
+    uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
+    uri += "/" + util.format_stanza_name(name)
+    msg = "Failed to delete data input in app=%s: %s://%s" % (
+        app_name, input_type, name)
+    content_request(uri, session_key, "DELETE", None, msg)
+
+
+def operate_data_input(splunkd_uri, session_key, owner, app_name,
+                       input_type, name, operation):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param input_type: name of the input type.
+                       if it is a script input, the input is "script",
+                       for modinput, say snow, the input is "snow"
+    :param name: The name of the input stanza to create.
+                 i.e. stanza [<input_type>://<name>] will be operated.
+    :param operation: must be "disable" or "enable"
+    """
+
+    assert operation in ("disable", "enable")
+    uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
+    uri += "/%s/%s" % (util.format_stanza_name(name), operation)
+    msg = "Failed to %s data input in app=%s: %s://%s" % (
+        operation, app_name, input_type, name)
+    content_request(uri, session_key, "POST", None, msg)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/knowledge_objects.py

@@ -0,0 +1,36 @@
+from . import request as req
+from ..common import xml_dom_parser as xdp
+
+
+class KnowledgeObjectManager(object):
+
+    def __init__(self, splunkd_uri, session_key):
+        self.splunkd_uri = splunkd_uri
+        self.session_key = session_key
+
+    def apps(self):
+        """
+        @return: a list of dict containing apps if successfuly otherwise
+        otherwise raise exceptions
+        """
+
+        uri = "{}/services/apps/local?count=0&offset=0".format(
+            self.splunkd_uri)
+        apps = self._do_request(uri, "GET", None, "Failed to get apps")
+        return apps
+
+    def indexes(self):
+        """
+        @return: a list of dict containing indexes if successfuly
+        otherwise raise exceptions
+        """
+
+        uri = "{}/services/data/indexes/?count=0&offset=0".format(
+            self.splunkd_uri)
+        indexes = self._do_request(uri, "GET", None, "Failed to get indexes")
+        return indexes
+
+    def _do_request(self, uri, method, payload, err_msg):
+        _, content = req.content_request(uri, self.session_key, method,
+                                         payload, err_msg)
+        return xdp.parse_conf_xml_dom(content)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/property_endpoints.py

@@ -0,0 +1,78 @@
+from .request import content_request
+from ..common import util
+
+
+PROPERTY_ENDPOINT = "%s/servicesNS/%s/%s/properties/%s"
+
+
+def _property_endpoint_ns(uri, owner, app, conf_name):
+    return PROPERTY_ENDPOINT % (uri, owner, app, conf_name)
+
+
+def create_properties(splunkd_uri, session_key, owner, app_name, conf_name,
+                      stanza):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param conf_name: the name of the conf file, e.g. "props"
+    :param stanza: stanza name, e.g. "aws:cloudtrail"
+    :return: None on success else raise exception
+    """
+
+    uri = _property_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
+    msg = "Properties: failed to create stanza=%s in conf=%s" % \
+          (stanza, conf_name)
+    payload = {"__stanza": stanza}
+    content_request(uri, session_key, "POST", payload, msg)
+
+
+def get_property(splunkd_uri, session_key, owner, app_name, conf_name,
+                 stanza, key):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param conf_name: the name of the conf file, e.g. "props"
+    :param stanza: stanza name, e.g. "aws:cloudtrail"
+    :param key: the property name
+    :return: the property value
+    """
+
+    uri = _property_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
+    uri += "/%s/%s" % (util.format_stanza_name(stanza), key)
+    msg = "Properties: failed to get conf=%s, stanza=%s, key=%s" % \
+          (conf_name, stanza, key)
+    return content_request(uri, session_key, "GET", None, msg)
+
+
+def update_properties(splunkd_uri, session_key, owner, app_name, conf_name,
+                      stanza, key_values):
+    """
+    :param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
+    :param session_key: splunkd session key
+    :param owner: the owner (ACL user), e.g. "-", "nobody"
+    :param app_name: the app"s name, e.g. "Splunk_TA_aws"
+    :param conf_name: the name of the conf file, e.g. "props"
+    :param stanza: stanza name, e.g. "aws:cloudtrail"
+    :param key_values: the key-value dict of the stanza
+    :return: raise exception when failed
+    """
+
+    uri = _property_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
+    uri += "/" + util.format_stanza_name(stanza)
+    msg = "Properties: failed to update conf=%s, stanza=%s" % \
+          (conf_name, stanza)
+
+    has_name = False
+    if "name" in key_values:
+        has_name = True
+        name = key_values["name"]
+        del key_values["name"]
+
+    content_request(uri, session_key, "POST", key_values, msg)
+
+    if has_name:
+        key_values["name"] = name

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/request.py

@@ -0,0 +1,44 @@
+from .. import rest
+from ..common import log
+
+
+class ConfRequestException(Exception):
+    pass
+
+
+class ConfNotExistsException(ConfRequestException):
+    pass
+
+
+class ConfExistsException(ConfRequestException):
+    pass
+
+
+def content_request(uri, session_key, method, payload, err_msg):
+    """
+    :return: response content if successful otherwise raise
+    ConfRequestException
+    """
+
+    resp, content = rest.splunkd_request(uri, session_key, method,
+                                         data=payload, retry=3)
+    if resp is None and content is None:
+        return None
+
+    if resp.status >= 200 and resp.status <= 204:
+        return content
+    else:
+        msg = "%s, status=%s, reason=%s, detail=%s" % (
+            err_msg, resp.status, resp.reason, content.decode('utf-8'))
+
+        if not (method == "GET" and resp.status == 404):
+            log.logger.error(msg)
+
+        if resp.status == 404:
+            raise ConfNotExistsException(msg)
+        if resp.status == 409:
+            raise ConfExistsException(msg)
+        else:
+            if content and "already exists" in content:
+                raise ConfExistsException(msg)
+            raise ConfRequestException(msg)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/ta_conf_manager.py

@@ -0,0 +1,211 @@
+"""
+This module hanles high level TA configuration related stuff
+"""
+
+import copy
+import os.path as op
+
+from . import conf_manager as conf
+from . import request as conf_req
+from .. import credentials as cred
+from ..common import util as utils
+
+
+class TAConfManager(object):
+
+    encrypted_token = "******"
+    reserved_keys = ("userName", "appName")
+
+    def __init__(self, conf_file, splunkd_uri, session_key, appname=None):
+        if appname is None:
+            appname = utils.get_appname_from_path(op.abspath(__file__))
+        self._conf_file = conf.conf_file2name(conf_file)
+        self._conf_mgr = conf.ConfManager(splunkd_uri, session_key,
+                                          app_name=appname)
+        self._cred_mgr = cred.CredentialManager(
+            splunkd_uri, session_key, app=appname,
+            owner="nobody", realm=appname)
+        self._keys = None
+
+    def set_appname(self, appname):
+        """
+        This are cases we need edit/remove/create confs in different app
+        context. call this interface to switch app context before manipulate
+        the confs in different app context
+        """
+
+        self._conf_mgr.set_appname(appname)
+        self._cred_mgr.set_appname(appname)
+
+    def _delete_reserved_keys(self, stanza):
+        new_stanza = copy.deepcopy(stanza)
+        for k in self.reserved_keys:
+            if k in new_stanza:
+                del new_stanza[k]
+        return new_stanza
+
+    def create(self, stanza):
+        """
+        @stanza: dick like object
+        {
+        "name": xxx,
+        "k1": v1,
+        "k2": v2,
+        ...
+        }
+        @return exception if failure
+        """
+
+        stanza = self._delete_reserved_keys(stanza)
+        encrypted_stanza = self._encrypt(stanza)
+        self._conf_mgr.create_stanza(self._conf_file,
+                                     encrypted_stanza["name"],
+                                     encrypted_stanza)
+
+    def update(self, stanza):
+        """
+        @stanza: dick like object
+        {
+        "name": xxx,
+        "k1": v1,
+        "k2": v2,
+        ...
+        }
+
+        @return: exception if failure
+        """
+
+        if not self._conf_mgr.stanza_exist(self._conf_file, stanza["name"]):
+            self.create(stanza)
+        else:
+            stanza = self._delete_reserved_keys(stanza)
+            encrypted_stanza = self._encrypt(stanza)
+            self._conf_mgr.update_properties(
+                self._conf_file, encrypted_stanza["name"], encrypted_stanza)
+
+    def delete(self, stanza_name):
+        """
+        @return: exception if failure
+        """
+
+        try:
+            stanza = self._conf_mgr.get_stanza(self._conf_file, stanza_name)
+        except conf_req.ConfNotExistsException:
+            return
+
+        self._delete_creds(stanza)
+        self._conf_mgr.delete_stanza(self._conf_file, stanza_name)
+
+    def get(self, stanza_name, return_acl=False):
+        """
+        @return: dict object if sucess otherwise raise exception
+        """
+
+        stanza = self._conf_mgr.get_stanza(self._conf_file, stanza_name,
+                                           ret_metadata=return_acl)
+        stanza = self._decrypt(stanza)
+        stanza["disabled"] = utils.is_true(stanza.get("disabled"))
+        return stanza
+
+    def all(self, filter_disabled=False, return_acl=True):
+        """
+        @return: a dict of dict objects if success
+        otherwise exception
+        """
+
+        results = {}
+        stanzas = self._conf_mgr.all_stanzas(self._conf_file,
+                                             ret_metadata=return_acl)
+        for stanza in stanzas:
+            stanza = self._decrypt(stanza)
+            stanza["disabled"] = utils.is_true(stanza.get("disabled"))
+            if filter_disabled and stanza["disabled"]:
+                continue
+            results[stanza["name"]] = stanza
+        return results
+
+    def reload(self):
+        self._conf_mgr.reload_conf(self._conf_file)
+
+    def set_encrypt_keys(self, keys):
+        """
+        :keys: a list keys of a stanza which need to be encrypted
+        for example: ["username", "password"]
+        """
+
+        self._keys = keys
+
+    def is_encrypted(self, stanza):
+        """
+        :stanza: dict object
+        return True if the values of encrypt keys equals self.encrypted_token
+        otherwise return False
+        """
+
+        if self._keys is None:
+            return False
+
+        for k in stanza.iterkeys():
+            if k in self._keys:
+                if stanza.get(k) == self.encrypted_token:
+                    return True
+        return False
+
+    def _encrypt(self, stanza):
+        """
+        :stanza: if self._keys are in stanza, encrypt the values of the key
+        and then mask the value to self.encrypted_token
+        """
+
+        if self._keys is None:
+            return stanza
+
+        stanza_to_be_encrypted = {}
+        for key in self._keys:
+            if key in stanza:
+                stanza_to_be_encrypted[key] = stanza[key]
+
+        if stanza_to_be_encrypted:
+            self._cred_mgr.update({stanza["name"]: stanza_to_be_encrypted})
+            encrypted_stanza = copy.deepcopy(stanza)
+            for key in stanza_to_be_encrypted.iterkeys():
+                encrypted_stanza[key] = self.encrypted_token
+            return encrypted_stanza
+        return stanza
+
+    def _decrypt(self, stanza):
+        """
+        :stanza: if there are keys in self._keys in stanza and if the values of
+        the keys are self.encrypted_token, decrypt the value
+        """
+
+        if self._keys is None:
+            return stanza
+
+        stanza_name = stanza["name"]
+        clear_password = None
+        for key in self._keys:
+            if key in stanza and stanza[key] == self.encrypted_token:
+                clear_password = self._cred_mgr.get_clear_password(
+                    stanza_name)
+                break
+
+        if clear_password:
+            for key in self._keys:
+                if key in clear_password[stanza_name]:
+                    stanza[key] = clear_password[stanza_name][key]
+        return stanza
+
+    def _delete_creds(self, stanza):
+        """
+        :stanza: if there are keys of self._keys and the keys are in stanza,
+        delete the encrypted creds
+        """
+
+        if self._keys is None:
+            return
+
+        for key in self._keys:
+            if key in stanza:
+                self._cred_mgr.delete(stanza["name"])
+                break

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/credentials.py

@@ -0,0 +1,337 @@
+"""
+Handles credentials related stuff
+"""
+
+import re
+import xml.dom.minidom as xdm
+
+from . import rest
+from .common import util
+from .common import xml_dom_parser as xdp
+
+# Splunk can only encrypt string when length <=255
+SPLUNK_CRED_LEN_LIMIT = 255
+
+
+class CredException(Exception):
+    pass
+
+
+class CredNotFound(CredException):
+    """
+    Credential information not exists
+    """
+    pass
+
+
+def create_credential_manager(username, password, splunkd_uri,
+                              app, owner, realm):
+    session_key = CredentialManager.get_session_key(
+        username, password, splunkd_uri)
+    return CredentialManager(splunkd_uri, session_key, app, owner, realm)
+
+
+class CredentialManager(object):
+    """
+    Credential related interfaces
+    """
+
+    def __init__(self, splunkd_uri, session_key,
+                 app="-", owner="nobody", realm=None):
+        """
+        :app: when creating/upating/deleting app is required
+        """
+
+        self._app = app
+        self._splunkd_uri = splunkd_uri
+        self._owner = owner
+        self._sep = "``splunk_cred_sep``"
+
+        if realm:
+            self._realm = realm
+        else:
+            self._realm = app
+
+        self._session_key = session_key
+
+    def set_appname(self, app):
+        """
+        This are cases we need edit/remove/create confs in different app
+        context. call this interface to switch app context before manipulate
+        the confs in different app context
+        """
+
+        self._app = app
+
+    @staticmethod
+    def get_session_key(username, password,
+                        splunkd_uri="https://localhost:8089"):
+        """
+        Get session key by using login username and passwrod
+        :return: session_key if successful, None if failed
+        """
+
+        eid = "".join((splunkd_uri, "/services/auth/login"))
+        postargs = {
+            "username": username,
+            "password": password,
+        }
+
+        response, content = rest.splunkd_request(
+            eid, None, method="POST", data=postargs)
+
+        if response is None and content is None:
+            raise CredException("Get session key failed.")
+
+        xml_obj = xdm.parseString(content)
+        session_nodes = xml_obj.getElementsByTagName("sessionKey")
+        if not session_nodes:
+            raise CredException("Invalid username or password.")
+        session_key = session_nodes[0].firstChild.nodeValue
+        if not session_key:
+            raise CredException("Get session key failed.")
+        return session_key
+
+    def update(self, stanza):
+        """
+        Update or Create credentials based on the stanza
+        :stanza: nested dict object. The outlayer keys are stanza name, and
+                 inner dict is user/pass key/value pair to be encrypted
+         {
+         "stanza_name": {"tommy": "tommypasswod", "jerry": "jerrypassword"}
+         }
+        :return: raise on failure
+        """
+
+        for name, encr_dict in stanza.items():
+            encrypts = []
+            for key, val in encr_dict.items():
+                encrypts.append(key)
+                encrypts.append(val)
+            self._update(name, self._sep.join(encrypts))
+
+    def _update(self, name, str_to_encrypt):
+        """
+        Update the string for the name.
+        :return: raise on failure
+        """
+
+        self.delete(name)
+
+        if len(str_to_encrypt) <= SPLUNK_CRED_LEN_LIMIT:
+            self._create(name, str_to_encrypt)
+            return
+
+        # split the str_to_encrypt when len > 255
+        length = SPLUNK_CRED_LEN_LIMIT
+        i = 0
+        while length < len(str_to_encrypt) + SPLUNK_CRED_LEN_LIMIT:
+            curr_str = str_to_encrypt[length - SPLUNK_CRED_LEN_LIMIT:length]
+            length += SPLUNK_CRED_LEN_LIMIT
+
+            stanza_name = self._sep.join((name, str(i)))
+            self._create(stanza_name, curr_str)
+            i += 1
+
+    def _create(self, name, str_to_encrypt):
+        """
+        Create a new stored credential.
+        :return: raise on failure
+        """
+
+        payload = {
+            "name": name,
+            "password": str_to_encrypt,
+            "realm": self._realm,
+        }
+
+        endpoint = self._get_endpoint(name)
+        resp, content = rest.splunkd_request(endpoint, self._session_key,
+                                             method="POST", data=payload)
+        if not resp or resp.status not in (200, 201, "200", "201"):
+            raise CredException("Failed to encrypt username {}".format(name))
+
+    def delete(self, name, throw=False):
+        """
+        Delete the encrypted entry
+        """
+
+        try:
+            self._delete(name, throw=True)
+        except CredNotFound:
+            # try to delete the split stanzas
+            try:
+                stanzas = self._get_all_passwords()
+            except Exception:
+                raise
+
+            ent_regx = "%s:(%s%s\d+):" % (self._realm, name, self._sep)
+            ent_pattern = re.compile(ent_regx)
+            for stanza in stanzas:
+                stanza_name = stanza.get("name")
+                match = ent_pattern.match(stanza_name)
+                if match:
+                    try:
+                        delete_name = match.group(1)
+                        self._delete(delete_name, throw=True)
+                    except CredNotFound:
+                        pass
+                    except CredException:
+                        raise
+        except CredException:
+            raise
+
+    def _delete(self, name, throw=False):
+        """
+        Delete the encrypted entry
+        """
+
+        endpoint = self._get_endpoint(name)
+        response, content = rest.splunkd_request(
+            endpoint, self._session_key, method="DELETE")
+
+        if response is not None and response.status in (404, "404"):
+            if throw:
+                raise CredNotFound(
+                    "Credential stanza not exits - {}".format(name))
+        elif not response or response.status not in (200, 201, "200", "201"):
+            if throw:
+                raise CredException(
+                    "Failed to delete credential stanza {}".format(name))
+
+    def get_all_passwords(self):
+        results = {}
+        all_stanzas = self._get_all_passwords()
+        for stanza in all_stanzas:
+            name = stanza.get("name")
+            match = re.match(r"(.+){}(\d+)".format(self._sep), name)
+            if match:
+                actual_name = match.group(1) + ":"
+                index = int(match.group(2))
+                if results.get(actual_name):
+                    exist_stanza = results.get(actual_name)
+                else:
+                    exist_stanza = stanza
+                    exist_stanza['name'] = actual_name
+                    exist_stanza['username'] = \
+                        exist_stanza['username'].split(self._sep)[0]
+                    exist_stanza['clears'] = {}
+                    exist_stanza['encrs'] = {}
+
+                try:
+                    exist_stanza['clears'][index] = stanza.get('clear_password')
+                    exist_stanza['encrs'][index] = stanza.get('encr_password')
+                except KeyError:
+                    exist_stanza['clears'] = {}
+                    exist_stanza['encrs'] = {}
+                    exist_stanza['clears'][index] = stanza.get('clear_password')
+                    exist_stanza['encrs'][index] = stanza.get('encr_password')
+
+                results[actual_name] = exist_stanza
+
+            else:
+                results[name] = stanza
+
+        # merge the stanzas by index
+        for name, stanza in results.items():
+            field_clear = stanza.get('clears')
+            field_encr = stanza.get('encrs')
+            if isinstance(field_clear, dict):
+                clear_password = ""
+                encr_password = ""
+                for index in sorted(field_clear.keys()):
+                    clear_password += field_clear.get(index)
+                    encr_password += field_encr.get(index)
+                stanza['clear_password'] = clear_password
+                stanza['encr_password'] = encr_password
+
+                del stanza['clears']
+                del stanza['encrs']
+        return results.values()
+
+    def _get_all_passwords(self):
+        """
+        :return: a list of dict when successful, None when failed.
+        the dict at least contains
+        {
+            "realm": xxx,
+            "username": yyy,
+            "clear_password": zzz,
+        }
+        """
+
+        endpoint = self._get_endpoint()
+        response, content = rest.splunkd_request(
+            endpoint, self._session_key, method="GET")
+        if response and response.status in (200, 201, "200", "201") and content:
+            return xdp.parse_conf_xml_dom(content)
+        raise CredException("Failed to get credentials")
+
+    def get_clear_password(self, name=None):
+        """
+        :return: clear password(s)
+        {
+        stanza_name: {"user": pass}
+        }
+        """
+
+        return self._get_credentials("clear_password", name)
+
+    def get_encrypted_password(self, name=None):
+        """
+        :return: encyrpted password(s)
+        """
+
+        return self._get_credentials("encr_password", name)
+
+    def _get_credentials(self, prop, name=None):
+        """
+        :return: clear or encrypted password for specified realm, user
+        """
+
+        all_stanzas = self.get_all_passwords()
+        results = {}
+
+        for stanza in all_stanzas:
+            if name and not stanza.get("name").endswith(":" + name + ":"):
+                continue
+            if stanza.get("realm") == self._realm:
+                values = stanza[prop].split(self._sep)
+                if len(values) % 2 == 1:
+                    continue
+                result = {values[i]: values[i + 1]
+                          for i in range(0, len(values), 2)}
+                results[stanza.get("username")] = result
+        return results
+
+    @staticmethod
+    def _build_name(realm, name):
+        return util.format_stanza_name(
+            "".join((CredentialManager._escape_string(realm), ":",
+                     CredentialManager._escape_string(name), ":")))
+
+    @staticmethod
+    def _escape_string(string_to_escape):
+        r"""
+        Splunk secure credential storage actually requires a custom style of
+        escaped string where all the :'s are escaped by a single \.
+        But don't escape the control : in the stanza name.
+        """
+
+        return string_to_escape.replace(":", "\\:")
+
+    def _get_endpoint(self, name=None, query=False):
+        app = self._app
+        owner = self._owner
+        if query:
+            app = "-"
+            owner = "-"
+
+        if name:
+            realm_user = self._build_name(self._realm, name)
+            rest_endpoint = "{}/servicesNS/{}/{}/storage/passwords/{}".format(
+                self._splunkd_uri, owner, app, realm_user)
+        else:
+            rest_endpoint = "{}/servicesNS/{}/{}/storage/passwords?count=-1" \
+                            "".format(self._splunkd_uri, owner, app)
+        return rest_endpoint

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/event_writer.py

@@ -0,0 +1,84 @@
+import Queue
+import multiprocessing
+import threading
+import sys
+from collections import Iterable
+from .common import log
+
+
+class EventWriter(object):
+
+    def __init__(self, process_safe=False):
+        if process_safe:
+            self._mgr = multiprocessing.Manager()
+            self._event_queue = self._mgr.Queue(1000)
+        else:
+            self._event_queue = Queue.Queue(1000)
+        self._event_writer = threading.Thread(target=self._do_write_events)
+        self._event_writer.daemon = True
+        self._started = False
+        self._exception = False
+
+    def start(self):
+        if self._started:
+            return
+        self._started = True
+
+        self._event_writer.start()
+        log.logger.info("Event writer started.")
+
+    def tear_down(self):
+        if not self._started:
+            return
+        self._started = False
+
+        self._event_queue.put(None)
+        self._event_writer.join()
+        log.logger.info("Event writer stopped.")
+
+    def isopen(self):
+        return self._started and (not self._exception)
+
+    def write_events(self, events):
+        if not self.isopen():
+            return False
+        if events is None:
+            return True
+        self._event_queue.put(events)
+        return True
+
+    def _do_write_events(self):
+        event_queue = self._event_queue
+        write = sys.stdout.write
+        got_shutdown_signal = False
+
+        while 1:
+            try:
+                event = event_queue.get(timeout=3)
+                if event is not None:
+                    if isinstance(event, basestring):
+                        write(event)
+                    elif isinstance(event, Iterable):
+                        for evt in event:
+                            write(evt)
+                else:
+                    log.logger.info("Event writer got tear down signal")
+                    got_shutdown_signal = True
+            except Queue.Empty:
+                # We need drain the queue before shutdown
+                # timeout means empty for now
+                if got_shutdown_signal:
+                    log.logger.info("Event writer is going to exit...")
+                    break
+                else:
+                    continue
+            except Exception:
+                log.logger.exception("EventWriter encounter exception which may"
+                                     "cause data loss, queue leftsize={"
+                                     "}".format(
+                                     event_queue.qsize()))
+                self._exception = True
+                break
+
+        log.logger.info("Event writer stopped, queue leftsize={}".format(
+            event_queue.qsize()))

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/file_monitor.py

@@ -0,0 +1,56 @@
+import os.path as op
+import traceback
+
+from .common import log
+
+
+class FileMonitor(object):
+
+    def __init__(self, callback, files):
+        """
+        :files: files to be monidtored with full path
+        """
+
+        self._callback = callback
+        self._files = files
+
+        self.file_mtimes = {
+            file_name: None for file_name in self._files
+        }
+        for k in self.file_mtimes:
+            if not op.exists(k):
+                continue
+
+            try:
+                if not op.exists(k):
+                    continue
+                self.file_mtimes[k] = op.getmtime(k)
+            except OSError:
+                log.logger.error("Getmtime for %s, failed: %s",
+                                 k, traceback.format_exc())
+
+    def __call__(self):
+        return self.check_changes()
+
+    def check_changes(self):
+        log.logger.debug("Checking files=%s", self._files)
+        file_mtimes = self.file_mtimes
+        changed_files = []
+        for f, last_mtime in file_mtimes.iteritems():
+            try:
+                if not op.exists(f):
+                    continue
+
+                current_mtime = op.getmtime(f)
+                if current_mtime != last_mtime:
+                    file_mtimes[f] = current_mtime
+                    changed_files.append(f)
+                    log.logger.info("Detect %s has changed", f)
+            except OSError:
+                pass
+
+        if changed_files:
+            if self._callback:
+                self._callback(changed_files)
+            return True
+        return False

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/kv_client.py

@@ -0,0 +1,202 @@
+import re
+import json
+from xml.etree import cElementTree as et
+
+from . import rest as rest
+
+
+class KVException(Exception):
+    pass
+
+
+class KVAlreadyExists(KVException):
+    pass
+
+
+class KVNotExists(KVException):
+    pass
+
+
+class KVClient(object):
+
+    def __init__(self, splunkd_host, session_key):
+        self._splunkd_host = splunkd_host
+        self._session_key = session_key
+
+    def create_collection(self, collection, app, owner="nobody"):
+        """
+        :collection: collection name
+        :return: None if successful otherwise KV exception thrown
+        """
+
+        assert collection
+        assert app
+
+        uri = self._get_config_endpoint(app, owner)
+        data = {
+            "name": collection
+        }
+        self._do_request(uri, "POST", data)
+
+    def list_collection(self, collection=None, app=None, owner="nobody"):
+        """
+        :collection: collection name. When euqals "None", return all
+        collections in the system.
+        :return: a list containing the connection names if successful, throws
+        KVNotExists if no such colection or other exception if other error
+        happened
+        """
+
+        uri = self._get_config_endpoint(app, owner, collection)
+
+        content = self._do_request(uri, method="GET")
+        m = re.search(r'xmlns="([^"]+)"', content)
+        path = "./entry/title"
+        if m:
+            ns = m.group(1)
+            path = "./{%s}entry/{%s}title" % (ns, ns)
+
+        collections = et.fromstring(content)
+        return [node.text for node in collections.iterfind(path)]
+
+    def delete_collection(self, collection, app, owner="nobody"):
+        """
+        :collection: collection name to be deleted
+        :return: None if successful otherwise throw KVNotExists exception if
+        the collection doesn't exist in the system or other exception if other
+        error happened
+        """
+
+        assert collection
+
+        uri = self._get_config_endpoint(app, owner, collection)
+        self._do_request(uri, method="DELETE")
+
+    def insert_collection_data(self, collection, data, app, owner="nobody"):
+        """
+        :collection: collection name
+        :data: dict like key values to be inserted and attached to
+        this collection
+        :return: {"_key": "key_id"} when successful, clients can use this
+        key to do query/delete/update, throws KV exceptions when failed
+        """
+
+        assert collection
+        assert data is not None
+        assert app
+
+        uri = self._get_data_endpoint(app, owner, collection)
+        key = self._do_request(uri, "POST", data,
+                               content_type="application/json")
+        return json.loads(key)
+
+    def delete_collection_data(self, collection, key_id, app, owner="nobody"):
+        """
+        :collection: collection name
+        :key_id: key id returned when creation. If None, delete all data
+        associated with this collection
+        :return: None if successful otherwise throws KV exception
+        """
+
+        assert collection
+
+        uri = self._get_data_endpoint(app, owner, collection, key_id)
+        self._do_request(uri, "DELETE", content_type="application/json")
+
+    def update_collection_data(self, collection, key_id, data,
+                               app, owner="nobody"):
+        """
+        :collection: collection name
+        :key_id: key id returned when creation
+        :return: key id if successful otherwise throws KV exception
+        """
+
+        assert collection
+        assert key_id
+        assert app
+
+        uri = self._get_data_endpoint(app, owner, collection, key_id)
+        k = self._do_request(uri, "POST", data,
+                             content_type="application/json")
+        return json.loads(k)
+
+    def get_collection_data(self, collection, key_id, app, owner="nobody"):
+        """
+        :collection: collection name
+        :key_id: key id returned when creation. If None, get all data
+        associated with this collection
+        :return: when key_id is not None, return key values if
+        successful. when key_id is None, return a list of key values if
+        sucessful. Throws KV exception if failure
+        """
+
+        assert collection
+
+        uri = self._get_data_endpoint(app, owner, collection, key_id)
+        k = self._do_request(uri, "GET")
+        return json.loads(k)
+
+    def _do_request(self, uri, method, data=None,
+                    content_type="application/x-www-form-urlencoded"):
+        headers = {"Content-Type": content_type}
+
+        resp, content = rest.splunkd_request(uri, self._session_key,
+                                             method, headers, data)
+        if resp is None and content is None:
+            raise KVException("Failed uri={0}, data={1}".format(uri, data))
+
+        if resp.status in (200, 201):
+            return content
+        elif resp.status == 409:
+            raise KVAlreadyExists("{0}-{1} already exists".format(uri, data))
+        elif resp.status == 404:
+            raise KVNotExists("{0}-{1} not exists".format(uri, data))
+        else:
+            raise KVException("Failed to {0} {1}, reason={2}".format(
+                method, uri, resp.reason))
+
+    def _get_config_endpoint(self, app, owner, collection=None):
+        uri = "{0}/servicesNS/{1}/{2}/storage/collections/config"
+        return self._do_get_endpoint(app, owner, collection, None, uri)
+
+    def _get_data_endpoint(self, app, owner, collection, key_id=None):
+        uri = "{0}/servicesNS/{1}/{2}/storage/collections/data"
+        return self._do_get_endpoint(app, owner, collection, key_id, uri)
+
+    def _do_get_endpoint(self, app, owner, collection, key_id, uri_template):
+        if not app:
+            app = "-"
+
+        if not owner:
+            owner = "-"
+
+        uri = uri_template.format(self._splunkd_host, owner, app)
+
+        if collection is not None:
+            uri += "/{0}".format(collection)
+            if key_id is not None:
+                uri += "/{0}".format(key_id)
+        return uri
+
+
+def create_collection(kv_client, collection, appname):
+    not_exists = False
+    try:
+        res = kv_client.list_collection(collection, appname)
+    except KVNotExists:
+        not_exists = True
+    except Exception:
+        not_exists = True
+
+    if not_exists or not res:
+        for i in xrange(3):
+            try:
+                kv_client.create_collection(collection, appname)
+            except KVAlreadyExists:
+                return
+            except Exception as e:
+                ex = e
+            else:
+                return
+        else:
+            raise ex

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/modinput.py

@@ -0,0 +1,147 @@
+import sys
+import subprocess
+import traceback
+
+from . import splunk_platform as sp
+from .common import log
+
+
+def _parse_modinput_configs(root, outer_block, inner_block):
+    """
+    When user splunkd spawns modinput script to do config check or run
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <input>
+      <server_host>localhost.localdomain</server_host>
+      <server_uri>https://127.0.0.1:8089</server_uri>
+      <session_key>xxxyyyzzz</session_key>
+      <checkpoint_dir>ckpt_dir</checkpoint_dir>
+      <configuration>
+        <stanza name="snow://alm_asset">
+          <param name="duration">60</param>
+            <param name="host">localhost.localdomain</param>
+            <param name="index">snow</param>
+            <param name="priority">10</param>
+        </stanza>
+        ...
+      </configuration>
+    </input>
+
+    When user create an stanza through data input on WebUI
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <items>
+      <server_host>localhost.localdomain</server_host>
+      <server_uri>https://127.0.0.1:8089</server_uri>
+      <session_key>xxxyyyzzz</session_key>
+      <checkpoint_dir>ckpt_dir</checkpoint_dir>
+      <item name="abc">
+        <param name="duration">60</param>
+        <param name="exclude"></param>
+        <param name="host">localhost.localdomain</param>
+        <param name="index">snow</param>
+        <param name="priority">10</param>
+      </item>
+    </items>
+    """
+
+    confs = root.getElementsByTagName(outer_block)
+    if not confs:
+        log.logger.error("Invalid config, missing %s section", outer_block)
+        raise Exception("Invalid config, missing %s section".format(
+            outer_block
+        ))
+
+    configs = []
+    stanzas = confs[0].getElementsByTagName(inner_block)
+    for stanza in stanzas:
+        config = {}
+        stanza_name = stanza.getAttribute("name")
+        if not stanza_name:
+            log.logger.error("Invalid config, missing name")
+            raise Exception("Invalid config, missing name")
+
+        config["name"] = stanza_name
+        params = stanza.getElementsByTagName("param")
+        for param in params:
+            name = param.getAttribute("name")
+            if (name and param.firstChild and
+                    param.firstChild.nodeType == param.firstChild.TEXT_NODE):
+                config[name] = param.firstChild.data
+        configs.append(config)
+    return configs
+
+
+def parse_modinput_configs(config_str):
+    """
+    @config_str: modinput XML configuration feed by splunkd
+    @return: meta_config and stanza_config
+    """
+
+    import xml.dom.minidom as xdm
+
+    meta_configs = {
+        "server_host": None,
+        "server_uri": None,
+        "session_key": None,
+        "checkpoint_dir": None,
+    }
+    root = xdm.parseString(config_str)
+    doc = root.documentElement
+    for tag in meta_configs.iterkeys():
+        nodes = doc.getElementsByTagName(tag)
+        if not nodes:
+            log.logger.error("Invalid config, missing %s section", tag)
+            raise Exception("Invalid config, missing %s section", tag)
+
+        if (nodes[0].firstChild and
+                nodes[0].firstChild.nodeType == nodes[0].TEXT_NODE):
+            meta_configs[tag] = nodes[0].firstChild.data
+        else:
+            log.logger.error("Invalid config, expect text ndoe")
+            raise Exception("Invalid config, expect text ndoe")
+
+    if doc.nodeName == "input":
+        configs = _parse_modinput_configs(doc, "configuration", "stanza")
+    else:
+        configs = _parse_modinput_configs(root, "items", "item")
+    return meta_configs, configs
+
+
+def get_modinput_configs_from_cli(modinput, modinput_stanza=None):
+    """
+    @modinput: modinput name
+    @modinput_stanza: modinput stanza name, for multiple instance only
+    """
+
+    assert modinput
+
+    splunkbin = sp.get_splunk_bin()
+    cli = [splunkbin, "cmd", "splunkd", "print-modinput-config", modinput]
+    if modinput_stanza:
+        cli.append(modinput_stanza)
+
+    out, err = subprocess.Popen(cli, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE).communicate()
+    if err:
+        log.logger.error("Failed to get modinput configs with error: %s", err)
+        return None, None
+    else:
+        return parse_modinput_configs(out)
+
+
+def get_modinput_config_str_from_stdin():
+    """
+    Get modinput from stdin which is feed by splunkd
+    """
+
+    try:
+        return sys.stdin.read(5000)
+    except Exception:
+        log.logger.error(traceback.format_exc())
+        raise
+
+
+def get_modinput_configs_from_stdin():
+    config_str = get_modinput_config_str_from_stdin()
+    return parse_modinput_configs(config_str)

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/orphan_process_monitor.py

@@ -0,0 +1,65 @@
+import os
+import threading
+import time
+import traceback
+
+from ..splunktalib.common import log
+
+
+class OrphanProcessChecker(object):
+
+    def __init__(self, callback=None):
+        """
+        Only work for Linux platform. On Windows platform, is_orphan is always
+        False
+        """
+
+        if os.name == "nt":
+            self._ppid = 0
+        else:
+            self._ppid = os.getppid()
+        self._callback = callback
+
+    def is_orphan(self):
+        if os.name == "nt":
+            return False
+        res = self._ppid != os.getppid()
+        if res:
+            log.logger.warn("Process=%s has become orphan", os.getpid())
+        return res
+
+    def check_orphan(self):
+        res = self.is_orphan()
+        if res and self._callback:
+            self._callback()
+        return res
+
+
+class OrphanProcessMonitor(object):
+
+    def __init__(self, callback):
+        self._checker = OrphanProcessChecker(callback)
+        self._thr = threading.Thread(target=self._do_monitor)
+        self._thr.daemon = True
+        self._started = False
+
+    def start(self):
+        if self._started:
+            return
+        self._started = True
+
+        self._thr.start()
+
+    def stop(self):
+        self._started = False
+
+    def _do_monitor(self):
+        while self._started:
+            try:
+                res = self._checker.check_orphan()
+                if res:
+                    break
+                time.sleep(1)
+            except Exception:
+                log.logger.error("Failed to monitor orphan process, reason=%s",
+                                 traceback.format_exc())

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/rest.py

@@ -0,0 +1,122 @@
+import urllib
+import json
+from traceback import format_exc
+
+from .common import util as scu
+from .common import log as log
+
+
+from httplib2 import (socks, ProxyInfo, Http)
+
+
+def splunkd_request(splunkd_uri, session_key, method="GET",
+                    headers=None, data=None, timeout=30, retry=1):
+    """
+    :return: httplib2.Response and content
+    """
+
+    headers = headers if headers is not None else {}
+    headers["Authorization"] = "Splunk {0}".format(session_key)
+    content_type = headers.get("Content-Type")
+    if not content_type:
+        content_type = headers.get("content-type")
+
+    if not content_type:
+        content_type = "application/x-www-form-urlencoded"
+        headers["Content-Type"] = content_type
+
+    if data is not None:
+        if content_type == "application/json":
+            data = json.dumps(data)
+        else:
+            data = urllib.urlencode(data)
+
+    http = Http(timeout=timeout, disable_ssl_certificate_validation=True)
+    msg_temp = "Failed to send rest request=%s, errcode=%s, reason=%s"
+    resp, content = None, None
+    for _ in range(retry):
+        try:
+            resp, content = http.request(splunkd_uri, method=method,
+                                         headers=headers, body=data)
+        except Exception:
+            log.logger.error(msg_temp, splunkd_uri, "unknown", format_exc())
+        else:
+            if resp.status not in (200, 201):
+                if not (method == "GET" and resp.status == 404):
+                    log.logger.debug(msg_temp, splunkd_uri, resp.status,
+                                     code_to_msg(resp, content))
+            else:
+                return resp, content
+    else:
+        return resp, content
+
+
+def code_to_msg(resp, content):
+    code_msg_tbl = {
+        400: "Request error. reason={}".format(content),
+        401: "Authentication failure, invalid access credentials.",
+        402: "In-use license disables this feature.",
+        403: "Insufficient permission.",
+        404: "Requested endpoint does not exist.",
+        409: "Invalid operation for this endpoint. reason={}".format(content),
+        500: "Unspecified internal server error. reason={}".format(content),
+        503: ("Feature is disabled in the configuration file. "
+              "reason={}".format(content)),
+    }
+
+    return code_msg_tbl.get(resp.status, content)
+
+
+def build_http_connection(config, timeout=120, disable_ssl_validation=False):
+    """
+    :config: dict like, proxy and account information are in the following
+             format {
+                 "username": xx,
+                 "password": yy,
+                 "proxy_url": zz,
+                 "proxy_port": aa,
+                 "proxy_username": bb,
+                 "proxy_password": cc,
+                 "proxy_type": http,http_no_tunnel,sock4,sock5,
+                 "proxy_rdns": 0 or 1,
+             }
+    :return: Http2.Http object
+    """
+
+    proxy_type_to_code = {
+        "http": socks.PROXY_TYPE_HTTP,
+        "http_no_tunnel": socks.PROXY_TYPE_HTTP_NO_TUNNEL,
+        "socks4": socks.PROXY_TYPE_SOCKS4,
+        "socks5": socks.PROXY_TYPE_SOCKS5,
+    }
+    if config.get("proxy_type") in proxy_type_to_code:
+        proxy_type = proxy_type_to_code[config["proxy_type"]]
+    else:
+        proxy_type = socks.PROXY_TYPE_HTTP
+
+    rdns = scu.is_true(config.get("proxy_rdns"))
+
+    proxy_info = None
+    if config.get("proxy_url") and config.get("proxy_port"):
+        if config.get("proxy_username") and config.get("proxy_password"):
+            proxy_info = ProxyInfo(proxy_type=proxy_type,
+                                   proxy_host=config["proxy_url"],
+                                   proxy_port=int(config["proxy_port"]),
+                                   proxy_user=config["proxy_username"],
+                                   proxy_pass=config["proxy_password"],
+                                   proxy_rdns=rdns)
+        else:
+            proxy_info = ProxyInfo(proxy_type=proxy_type,
+                                   proxy_host=config["proxy_url"],
+                                   proxy_port=int(config["proxy_port"]),
+                                   proxy_rdns=rdns)
+    if proxy_info:
+        http = Http(proxy_info=proxy_info, timeout=timeout,
+                    disable_ssl_certificate_validation=disable_ssl_validation)
+    else:
+        http = Http(timeout=timeout,
+                    disable_ssl_certificate_validation=disable_ssl_validation)
+
+    if config.get("username") and config.get("password"):
+        http.add_credentials(config["username"], config["password"])
+    return http

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/schedule/job.py

@@ -0,0 +1,90 @@
+import threading
+import time
+
+
+class Job(object):
+    """
+    Timer wraps the callback and timestamp related stuff
+    """
+
+    _ident = 0
+    _lock = threading.Lock()
+
+    def __init__(self, func, job_props, interval, when=None, job_id=None):
+        """
+        @job_props: dict like object
+        @func: execution function
+        @interval: execution interval
+        @when: seconds from epoch
+        @job_id: a unique id for the job
+        """
+
+        self._props = job_props
+        self._func = func
+        if when is None:
+            self._when = time.time()
+        else:
+            self._when = when
+        self._interval = interval
+
+        if job_id is not None:
+            self._id = job_id
+        else:
+            with Job._lock:
+                self._id = Job._ident + 1
+                Job._ident = Job._ident + 1
+        self._stopped = False
+
+    def ident(self):
+        return self._id
+
+    def get_interval(self):
+        return self._interval
+
+    def set_interval(self, interval):
+        self._interval = interval
+
+    def get_expiration(self):
+        return self._when
+
+    def set_initial_due_time(self, when):
+        if self._when is None:
+            self._when = when
+
+    def update_expiration(self):
+        self._when += self._interval
+
+    def get(self, key, default):
+        return self._props.get(key, default)
+
+    def get_props(self):
+        return self._props
+
+    def set_props(self, props):
+        self._props = props
+
+    def __cmp__(self, other):
+        if other is None:
+            return 1
+
+        self_k = (self.get_expiration(), self.ident())
+        other_k = (other.get_expiration(), other.ident())
+
+        if self_k == other_k:
+            return 0
+        elif self_k < other_k:
+            return -1
+        else:
+            return 1
+
+    def __eq__(self, other):
+        return isinstance(other, Job) and (self.ident() == other.ident())
+
+    def __call__(self):
+        self._func(self)
+
+    def stop(self):
+        self._stopped = True
+
+    def stopped(self):
+        return self._stopped

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/schedule/scheduler.py

@@ -0,0 +1,143 @@
+import threading
+from time import time
+import random
+import Queue
+from ..common import log
+
+
+class Scheduler(object):
+    """
+    A simple scheduler which schedules the periodic or once event
+    """
+
+    import sortedcontainers as sc
+
+    max_delay_time = 60
+
+    def __init__(self):
+        self._jobs = Scheduler.sc.SortedSet()
+        self._wakeup_q = Queue.Queue()
+        self._lock = threading.Lock()
+        self._thr = threading.Thread(target=self._do_jobs)
+        self._thr.deamon = True
+        self._started = False
+
+    def start(self):
+        """
+        Start the schduler which will start the internal thread for scheduling
+        jobs. Please do tear_down when doing cleanup
+        """
+
+        if self._started:
+            log.logger.info("Scheduler already started.")
+            return
+        self._started = True
+
+        self._thr.start()
+
+    def tear_down(self):
+        """
+        Stop the schduler which will stop the internal thread for scheduling
+        jobs.
+        """
+
+        if not self._started:
+            log.logger.info("Scheduler already tear down.")
+            return
+
+        self._wakeup_q.put(True)
+
+    def _do_jobs(self):
+        while 1:
+            (sleep_time, jobs) = self.get_ready_jobs()
+            self._do_execution(jobs)
+            try:
+                done = self._wakeup_q.get(timeout=sleep_time)
+            except Queue.Empty:
+                pass
+            else:
+                if done:
+                    break
+        self._started = False
+        log.logger.info("Scheduler exited.")
+
+    def get_ready_jobs(self):
+        """
+        @return: a 2 element tuple. The first element is the next ready
+                 duration. The second element is ready jobs list
+        """
+
+        now = time()
+        ready_jobs = []
+        sleep_time = 1
+
+        with self._lock:
+            job_set = self._jobs
+            total_jobs = len(job_set)
+            for job in job_set:
+                if job.get_expiration() <= now:
+                    ready_jobs.append(job)
+
+            if ready_jobs:
+                del job_set[:len(ready_jobs)]
+
+            for job in ready_jobs:
+                if job.get_interval() != 0 and not job.stopped():
+                    # repeated job, calculate next due time and enqueue
+                    job.update_expiration()
+                    job_set.add(job)
+
+            if job_set:
+                sleep_time = job_set[0].get_expiration() - now
+                if sleep_time < 0:
+                    log.logger.warn("Scheduler satuation, sleep_time=%s",
+                                    sleep_time)
+                    sleep_time = 0.1
+
+        if ready_jobs:
+            log.logger.info("Get %d ready jobs, next duration is %f, "
+                            "and there are %s jobs scheduling",
+                            len(ready_jobs), sleep_time, total_jobs)
+
+        ready_jobs.sort(key=lambda job: job.get("priority", 0), reverse=True)
+        return (sleep_time, ready_jobs)
+
+    def add_jobs(self, jobs):
+        with self._lock:
+            now = time()
+            job_set = self._jobs
+            for job in jobs:
+                delay_time = random.randrange(0, self.max_delay_time)
+                job.set_initial_due_time(now + delay_time)
+                job_set.add(job)
+        self._wakeup()
+
+    def update_jobs(self, jobs):
+        with self._lock:
+            job_set = self._jobs
+            for njob in jobs:
+                job_set.discard(njob)
+                job_set.add(njob)
+        self._wakeup()
+
+    def remove_jobs(self, jobs):
+        with self._lock:
+            job_set = self._jobs
+            for njob in jobs:
+                njob.stop()
+                job_set.discard(njob)
+        self._wakeup()
+
+    def number_of_jobs(self):
+        with self._lock:
+            return len(self._jobs)
+
+    def disable_randomization(self):
+        self.max_delay_time = 1
+
+    def _wakeup(self):
+        self._wakeup_q.put(None)
+
+    def _do_execution(self, jobs):
+        for job in jobs:
+            job()

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/setting.conf

@@ -0,0 +1,5 @@
+[global]
+process_size = 0
+thread_min_size = 4
+thread_max_size = 128
+task_queue_size = 1024

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/splunk_cluster.py

@@ -0,0 +1,53 @@
+from ..splunktalib import rest
+from ..splunktalib.common import xml_dom_parser as xdp
+
+
+def _do_rest(uri, session_key):
+    resp, content = rest.splunkd_request(uri, session_key)
+    if resp is None:
+        return None
+
+    if resp.status not in (200, 201):
+        return None
+
+    stanza_objs = xdp.parse_conf_xml_dom(content)
+    if not stanza_objs:
+        return None
+
+    return stanza_objs[0]
+
+
+class ServerInfo(object):
+
+    def __init__(self, splunkd_uri, session_key):
+        uri = "{}/services/server/info".format(splunkd_uri)
+        server_info = _do_rest(uri, session_key)
+        if server_info is None:
+            raise Exception("Failed to init ServerInfo")
+
+        self._server_info = server_info
+
+    def is_captain(self):
+        """
+        :return: True if splunkd_uri is captain otherwise False
+        """
+
+        return "shc_captain" in self._server_info["server_roles"]
+
+    def is_search_head(self):
+        for sh in ("search_head", "cluster_search_head"):
+            if sh in self._server_info["server_roles"]:
+                return True
+        return False
+
+    def is_shc_member(self):
+        server_roles = self._server_info['server_roles']
+        return any(
+            role in server_roles for role in ('shc_member', 'shc_captain')
+        )
+
+    def version(self):
+        return self._server_info["version"]
+
+    def to_dict(self):
+        return self._server_info

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/splunk_platform.py

@@ -0,0 +1,104 @@
+import os
+import os.path as op
+import subprocess
+from ConfigParser import ConfigParser
+from cStringIO import StringIO
+
+from .common import util as scu
+
+
+def make_splunkhome_path(parts):
+    """
+    create a path string by the several parts of the path
+    """
+
+    relpath = os.path.normpath(os.path.join(*parts))
+
+    basepath = os.environ["SPLUNK_HOME"]  # Assume SPLUNK_HOME env has been set
+
+    fullpath = os.path.normpath(os.path.join(basepath, relpath))
+
+    # Check that we haven't escaped from intended parent directories.
+    if os.path.relpath(fullpath, basepath)[0:2] == '..':
+        raise ValueError('Illegal escape from parent directory "%s": %s' %
+                         (basepath, fullpath))
+
+    return fullpath
+
+
+def get_splunk_bin():
+    if os.name == "nt":
+        splunk_bin = "splunk.exe"
+    else:
+        splunk_bin = "splunk"
+    return make_splunkhome_path(("bin", splunk_bin))
+
+
+def get_appname_from_path(absolute_path):
+    return scu.get_appname_from_path(absolute_path)
+
+
+def _get_merged_conf_raw(conf_name):
+    """
+    :conf_name: configure file name
+    :return: raw output of all contents for the same conf file
+    Note: it depends on SPLUNK_HOME env variable
+    """
+
+    assert conf_name
+
+    if conf_name.endswith(".conf"):
+        conf_name = conf_name[:-5]
+
+    # FIXME dynamically caculate SPLUNK_HOME
+    btool_cli = [op.join(os.environ["SPLUNK_HOME"], "bin", "btool"),
+                 conf_name, "list"]
+
+    try:
+        p = subprocess.Popen(btool_cli, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        out, err = p.communicate()
+    except OSError:
+        raise
+
+    return out
+
+
+def _get_conf_stanzas(conf_name):
+    """
+    :return: {stanza_name: stanza_configs}, dict of dict
+    """
+
+    res = _get_merged_conf_raw(conf_name)
+    res = StringIO(res)
+    parser = ConfigParser()
+    parser.optionxform = str
+    parser.readfp(res)
+    res = {}
+    for section in parser.sections():
+        res[section] = {item[0]: item[1] for item in parser.items(section)}
+    return res
+
+
+def get_splunkd_uri():
+    if "SPLUNKD_URI" in os.environ:
+        return os.environ["SPLUNKD_URI"]
+    else:
+        server_conf = _get_conf_stanzas("server")
+        if server_conf["sslConfig"]["enableSplunkdSSL"].lower() == "true":
+            http = "https://"
+        else:
+            http = "http://"
+
+        web_conf = _get_conf_stanzas("web")
+        host_port = web_conf["settings"]["mgmtHostPort"]
+        splunkd_uri = "{}{}".format(http, host_port)
+
+        if os.environ.get("SPLUNK_BINDIP"):
+            bip = os.environ["SPLUNK_BINDIP"]
+            port_idx = bip.rfind(":")
+            if port_idx > 0:
+                bip = bip[:port_idx]
+            port = host_port[host_port.rfind(":"):]
+            splunkd_uri = "{}{}{}".format(http, bip, port)
+        return splunkd_uri

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/state_store.py

@@ -0,0 +1,264 @@
+import json
+import os
+import os.path as op
+import time
+
+from ..splunktacollectorlib.common import log as stulog
+from ..splunktalib import kv_client as kvc
+from ..splunktalib.common import util
+
+
+def get_state_store(meta_configs,
+                    appname,
+                    collection_name="talib_states",
+                    use_kv_store=False,
+                    use_cache_file=True,
+                    max_cache_seconds=5):
+    # FIXME refactor this
+    if util.is_true(use_kv_store):
+        return StateStore(meta_configs, appname, collection_name)
+    if util.is_true(use_cache_file):
+        return CachedFileStateStore(meta_configs, appname, max_cache_seconds)
+    return FileStateStore(meta_configs, appname)
+
+
+class BaseStateStore(object):
+    def __init__(self, meta_configs, appname):
+        self._meta_configs = meta_configs
+        self._appname = appname
+
+    def update_state(self, key, states):
+        pass
+
+    def get_state(self, key):
+        pass
+
+    def delete_state(self, key):
+        pass
+
+    def close(self, key=None):
+        pass
+
+
+class StateStore(BaseStateStore):
+
+    def __init__(self, meta_configs, appname, collection_name="talib_states"):
+        """
+        :meta_configs: dict like and contains checkpoint_dir, session_key,
+         server_uri etc
+        :app_name: the name of the app
+        :collection_name: the collection name to be used.
+        Don"t use other method to visit the collection if you are using
+         StateStore to visit it.
+        """
+        super(StateStore, self).__init__(meta_configs, appname)
+
+        # State cache is a dict from _key to value
+        self._states_cache = {}
+        self._kv_client = None
+        self._collection = collection_name
+        self._kv_client = kvc.KVClient(meta_configs["server_uri"],
+                                       meta_configs["session_key"])
+        kvc.create_collection(self._kv_client, self._collection, self._appname)
+        self._load_states_cache()
+
+    def update_state(self, key, states):
+        """
+        :state: Any JSON serializable
+        :return: None if successful, otherwise throws exception
+        """
+
+        if key not in self._states_cache:
+            self._kv_client.insert_collection_data(
+                self._collection, {"_key": key, "value": json.dumps(states)},
+                self._appname)
+        else:
+            self._kv_client.update_collection_data(
+                self._collection, key, {"value": json.dumps(states)},
+                self._appname)
+        self._states_cache[key] = states
+
+    def get_state(self, key=None):
+        if key:
+            return self._states_cache.get(key, None)
+        return self._states_cache
+
+    def delete_state(self, key=None):
+        if key:
+            self._delete_state(key)
+        else:
+            [self._delete_state(_key) for _key in self._states_cache.keys()]
+
+    def _delete_state(self, key):
+        if key not in self._states_cache:
+            return
+
+        self._kv_client.delete_collection_data(
+            self._collection, key, self._appname)
+        del self._states_cache[key]
+
+    def _load_states_cache(self):
+        states = self._kv_client.get_collection_data(
+            self._collection, None, self._appname)
+        if not states:
+            return
+
+        for state in states:
+            if "value" in state:
+                value = state["value"]
+            else:
+                value = state
+
+            try:
+                value = json.loads(value)
+            except Exception:
+                pass
+
+            self._states_cache[state["_key"]] = value
+
+
+def _create_checkpoint_dir_if_needed(checkpoint_dir):
+    if os.path.isdir(checkpoint_dir):
+        return
+
+    stulog.logger.info(
+        "Checkpoint dir '%s' doesn't exist, try to create it",
+        checkpoint_dir)
+    try:
+        os.mkdir(checkpoint_dir)
+    except OSError:
+        stulog.logger.exception(
+            "Failure creating checkpoint dir '%s'", checkpoint_dir
+        )
+        raise Exception(
+            "Unable to create checkpoint dir '{}'".format(checkpoint_dir)
+        )
+
+
+class FileStateStore(BaseStateStore):
+    def __init__(self, meta_configs, appname):
+        """
+        :meta_configs: dict like and contains checkpoint_dir, session_key,
+        server_uri etc
+        """
+
+        super(FileStateStore, self).__init__(meta_configs, appname)
+
+    def update_state(self, key, states):
+        """
+        :state: Any JSON serializable
+        :return: None if successful, otherwise throws exception
+        """
+
+        checkpoint_dir = self._meta_configs["checkpoint_dir"]
+        _create_checkpoint_dir_if_needed(checkpoint_dir)
+
+        fname = op.join(checkpoint_dir, key)
+        with open(fname + ".new", "w") as jsonfile:
+            json.dump(states, jsonfile)
+
+        if op.exists(fname):
+            os.remove(fname)
+
+        os.rename(fname + ".new", fname)
+        # commented this to disable state cache for local file
+        # if key not in self._states_cache:
+        # self._states_cache[key] = {}
+        # self._states_cache[key] = states
+
+    def get_state(self, key):
+        fname = op.join(self._meta_configs["checkpoint_dir"], key)
+        if op.exists(fname):
+            with open(fname) as jsonfile:
+                state = json.load(jsonfile)
+                # commented this to disable state cache for local file
+                # self._states_cache[key] = state
+                return state
+        else:
+            return None
+
+    def delete_state(self, key):
+        fname = op.join(self._meta_configs["checkpoint_dir"], key)
+        if op.exists(fname):
+            os.remove(fname)
+
+
+class CachedFileStateStore(BaseStateStore):
+    def __init__(self, meta_configs, appname, max_cache_seconds=5):
+        """
+        :meta_configs: dict like and contains checkpoint_dir, session_key,
+        server_uri etc
+        """
+
+        super(CachedFileStateStore, self).__init__(meta_configs, appname)
+        self._states_cache = {} # item: time, dict
+        self._states_cache_lmd = {} #item: time, dict
+        self.max_cache_seconds = max_cache_seconds
+
+    def update_state(self, key, states):
+
+        now = time.time()
+        if key in self._states_cache:
+            last = self._states_cache_lmd[key][0]
+            if now - last >= self.max_cache_seconds:
+                self.update_state_flush(now, key, states)
+        else:
+            self.update_state_flush(now, key, states)
+        self._states_cache[key] = (now, states)
+
+    def update_state_flush(self, now, key, states):
+        """
+        :state: Any JSON serializable
+        :return: None if successful, otherwise throws exception
+        """
+        self._states_cache_lmd[key] = (now, states)
+        checkpoint_dir = self._meta_configs["checkpoint_dir"]
+
+        _create_checkpoint_dir_if_needed(checkpoint_dir)
+
+        fname = op.join(checkpoint_dir, key)
+
+        with open(fname + ".new", "w") as jsonfile:
+            json.dump(states, jsonfile)
+
+        if op.exists(fname):
+            os.remove(fname)
+
+        os.rename(fname + ".new", fname)
+
+    def get_state(self, key):
+        if key in self._states_cache:
+            return self._states_cache[key][1]
+
+        fname = op.join(self._meta_configs["checkpoint_dir"], key)
+        if op.exists(fname):
+            with open(fname) as jsonfile:
+                state = json.load(jsonfile)
+                now = time.time()
+                self._states_cache[key] = now, state
+                self._states_cache_lmd[key] = now, state
+                return state
+        else:
+            return None
+
+    def delete_state(self, key):
+        fname = op.join(self._meta_configs["checkpoint_dir"], key)
+        if op.exists(fname):
+            os.remove(fname)
+
+        if self._states_cache.get(key):
+            del self._states_cache[key]
+        if self._states_cache_lmd.get(key):
+            del self._states_cache_lmd[key]
+
+    def close(self, key=None):
+        if not key:
+            for k, (t, s) in self._states_cache.iteritems():
+                self.update_state_flush(t, k, s)
+            self._states_cache.clear()
+            self._states_cache_lmd.clear()
+        elif key in self._states_cache:
+            self.update_state_flush(self._states_cache[key][0], key,
+                                    self._states_cache[key][1])
+            del self._states_cache[key]
+            del self._states_cache_lmd[key]

package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/timer.py

@@ -0,0 +1,60 @@
+import threading
+
+
+class Timer(object):
+    """
+    Timer wraps the callback and timestamp related stuff
+    """
+
+    _ident = 0
+    _lock = threading.Lock()
+
+    def __init__(self, callback, when, interval, ident=None):
+        self._callback = callback
+        self._when = when
+        self._interval = interval
+
+        if ident is not None:
+            self._id = ident
+        else:
+            with Timer._lock:
+                self._id = Timer._ident + 1
+                Timer._ident = Timer._ident + 1
+
+    def get_interval(self):
+        return self._interval
+
+    def set_interval(self, interval):
+        self._interval = interval
+
+    def get_expiration(self):
+        return self._when
+
+    def set_initial_due_time(self, when):
+        self._when = when
+
+    def update_expiration(self):
+        self._when += self._interval
+
+    def __cmp__(self, other):
+        if other is None:
+            return 1
+
+        self_k = (self.get_expiration(), self.ident())
+        other_k = (other.get_expiration(), other.ident())
+
+        if self_k == other_k:
+            return 0
+        elif self_k < other_k:
+            return -1
+        else:
+            return 1
+
+    def __eq__(self, other):
+        return isinstance(other, Timer) and (self.ident() == other.ident())
+
+    def __call__(self):
+        self._callback()
+
+    def ident(self):
+        return self._id