admin/splunk-security_content
1
0
Fork 0
mirror of https://github.com/splunk/security_content synced 2026-06-08 17:32:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into readme_fix

Browse Source
This commit is contained in:
jzsplunk
2019-12-10 09:49:10 -08:00
parent fd049298b0 d6dfa714ef
commit 8769b07019
627 changed files with 57 additions and 179644 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+2 -2
View File
@@ -90,8 +90,8 @@ jobs:
source $BASH_ENV
go get -u github.com/raviqqe/liche
cd security-content
liche -r docs/
liche README.md
liche docs/stories_categories.md -v
liche README.md -v
build-sources:
executor: content-executor
steps:
README.md
+2 -2
View File
@@ -79,7 +79,7 @@ Install project dependecies and tests that run before content is committed:
##### CI Tools
Tools that help with testing CI jobs:
1. Install CircleCI [CLI Tool](https://circleci.com/docs/2.0/local-cli/#installation).
1. Install CircleCI [CLI Tool](https://circleci.com/docs/2.0/local-cli/).
2. To test a local change to CircleCI or build, make sure you are running Docker, then enter
`circleci local execute -e GITHUB_TOKEN=$GITHUB_TOKEN --branch <your branch>`.
@@ -99,7 +99,7 @@ To automatically generate docs from schema:
* note that [requirements.txt](https://github.com/splunk/security-content/blob/develop/requirements.txt) hardcodes the versions for packages we use [dependabot](https://dependabot.com/) to make sure we safely always upgrade to the latest versions.
## Support
Please use the [GitHub Issue Tracker](https://github.com/splunk/security_content/issues) to submit bugs or request features.
Please use the [GitHub Issue Tracker](https://github.com/splunk/security-content/issues) to submit bugs or request features.
If you have questions or need support, you can:
bin/generate.py
-4
View File
@@ -149,10 +149,6 @@ def identify_next_steps(detections, investigations):
next_steps += "Based on ESCU investigate recommendations:\\n%s\"}" % (investigations_output)
if has_phantom:
detection['recommended_actions'] = 'runphantomplaybook, escu_investigate'
else:
detection['recommended_actions'] = 'escu_investigate'
detection['next_steps'] = next_steps
enriched_detections.append(detection)
return enriched_detections
bin/jinja2_templates/savedsearches.j2
-2
View File
@@ -79,8 +79,6 @@ action.notable.param.severity = {{ detection.confidence }}
action.notable.param.drilldown_name = {{ correlation_rule.notable.drilldown_name }}
action.notable.param.drilldown_search = {{ correlation_rule.notable.drilldown_search }}
{% endif %}
action.notable.param.recommended_actions = {{ detection.recommended_actions }}
action.notable.param.next_steps = {{ detection.next_steps }}
{% if correlation_rule.risk is defined %}
action.risk = 1
action.risk.param._risk_object = {{ correlation_rule.risk.risk_object }}
package/README/savedsearches.conf.example
+1 -3
View File
@@ -30,8 +30,6 @@ action.notable.param.rule_description = Your AWS infrastructure was provisioned
action.notable.param.rule_title = AWS Provision Activity From $city$
action.notable.param.security_domain = endpoint
action.notable.param.severity = medium
action.notable.param.recommended_actions = escu_investigate
action.notable.param.next_steps = {"version": 1, "data": "Recommended following steps:\n\n1. [[action|escu_investigate]]: Based on ESCU investigate recommendations:\nESCU - Get All AWS Activity From IP Address\nESCU - Get All AWS Activity From City\n"}
action.risk = 1
action.risk.param._risk_object = dest
action.risk.param._risk_object_type = system
@@ -98,4 +96,4 @@ action.escu.known_false_positives =
disabled=true
schedule_window = auto
is_visible = false
search = | inputlookup interesting_processes_lookup | search note!=ESCU* | inputlookup append=T prohibitedProcesses_lookup | fillnull value=* dest dest_pci_domain | fillnull value=false is_required is_secure | fillnull value=true is_prohibited | outputlookup interesting_processes_lookup | stats count
search = | inputlookup interesting_processes_lookup | search note!=ESCU* | inputlookup append=T prohibitedProcesses_lookup | fillnull value=* dest dest_pci_domain | fillnull value=false is_required is_secure | fillnull value=true is_prohibited | outputlookup interesting_processes_lookup | stats count
package/appserver/static/analytic_story_details.js
+2 -7
View File
@@ -146,12 +146,7 @@ require([
$('.run-story').unbind('click');
$('.run-story').on('click', function(evt) {
splunkUtil.redirect_to('/app/DA-ESS-ContentUpdate/search', {
q: `| runstory story="${asName}" | table name, num_search_results, description, kill_chain_phases, mitre_attack`,
earliest: "-60m",
latest: "now",
}, window.open(), true);
window.open('https://github.com/splunk/analytic_story_execution');
});
let asSearch = splunkjs.mvc.Components.getInstance(epoch);
@@ -593,4 +588,4 @@ require([
return htmlTmpl;
}
});
});
package/bin/da_ess_contentupdate/alert_actions_base.py
-211
View File
@@ -1,211 +0,0 @@
import csv
import gzip
import sys
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.insert(0, make_splunkhome_path(["etc", "apps", "Splunk_SA_CIM", "lib"]))
import requests
from cim_actions import ModularAction
from logging_helper import get_logger
import logging
from splunk_aoblib.rest_helper import TARestHelper
from splunk_aoblib.setup_util import Setup_Util
class ModularAlertBase(ModularAction):
def __init__(self, ta_name, alert_name):
self._alert_name = alert_name
# self._logger_name = "modalert_" + alert_name
self._logger_name = alert_name + "_modalert"
self._logger = get_logger(self._logger_name)
super(ModularAlertBase, self).__init__(
sys.stdin.read(), self._logger, alert_name)
self.setup_util_module = None
self.setup_util = None
self.result_handle = None
self.ta_name = ta_name
self.splunk_uri = self.settings.get('server_uri')
self.setup_util = Setup_Util(self.splunk_uri, self.session_key, self._logger)
level = self.get_log_level()
if level:
self._logger.setLevel(level)
self.rest_helper = TARestHelper(self._logger)
def log_error(self, msg):
self.message(msg, 'failure', level=logging.ERROR)
def log_info(self, msg):
self.message(msg, 'success', level=logging.INFO)
def log_debug(self, msg):
self.message(msg, None, level=logging.DEBUG)
def log_warn(self, msg):
self.message(msg, None, level=logging.WARN)
def set_log_level(self, level):
self._logger.setLevel(level)
def get_param(self, param_name):
return self.configuration.get(param_name)
def get_global_setting(self, var_name):
return self.setup_util.get_customized_setting(var_name)
def get_user_credential(self, username):
'''
if the username exists, return
{
"username": username,
"password": credential
}
'''
return self.setup_util.get_credential_by_username(username)
@property
def log_level(self):
return self.get_log_level()
@property
def proxy(self):
return self.get_proxy()
def get_log_level(self):
return self.setup_util.get_log_level()
def get_proxy(self):
''' if the proxy setting is set. return a dict like
{
proxy_url: ... ,
proxy_port: ... ,
proxy_username: ... ,
proxy_password: ... ,
proxy_type: ... ,
proxy_rdns: ...
}
'''
return self.setup_util.get_proxy_settings()
def _get_proxy_uri(self):
uri = None
proxy = self.get_proxy()
if proxy and proxy.get('proxy_url') and proxy.get('proxy_type'):
uri = proxy['proxy_url']
if proxy.get('proxy_port'):
uri = '{0}:{1}'.format(uri, proxy.get('proxy_port'))
if proxy.get('proxy_username') and proxy.get('proxy_password'):
uri = '{0}://{1}:{2}@{3}/'.format(proxy['proxy_type'], proxy[
'proxy_username'], proxy['proxy_password'], uri)
else:
uri = '{0}://{1}'.format(proxy['proxy_type'], uri)
return uri
def send_http_request(self, url, method, parameters=None, payload=None, headers=None, cookies=None, verify=True, cert=None, timeout=None, use_proxy=True):
return self.rest_helper.send_http_request(url=url, method=method, parameters=parameters, payload=payload,
headers=headers, cookies=cookies, verify=verify, cert=cert,
timeout=timeout,
proxy_uri=self._get_proxy_uri() if use_proxy else None)
def build_http_connection(self, config, timeout=120,
disable_ssl_validation=False):
from httplib2 import (socks, ProxyInfo, Http)
"""
:config: dict like, proxy and account information are in the following
format {
"username": xx,
"password": yy,
"proxy_url": zz,
"proxy_port": aa,
"proxy_username": bb,
"proxy_password": cc,
"proxy_type": http,http_no_tunnel,sock4,sock5,
"proxy_rdns": 0 or 1,
}
:return: Http2.Http object
"""
if not config:
config = {}
proxy_type_to_code = {
"http": socks.PROXY_TYPE_HTTP,
"http_no_tunnel": socks.PROXY_TYPE_HTTP_NO_TUNNEL,
"socks4": socks.PROXY_TYPE_SOCKS4,
"socks5": socks.PROXY_TYPE_SOCKS5,
}
if config.get("proxy_type") in proxy_type_to_code:
proxy_type = proxy_type_to_code[config["proxy_type"]]
else:
proxy_type = socks.PROXY_TYPE_HTTP
rdns = config.get("proxy_rdns")
proxy_info = None
if config.get("proxy_url") and config.get("proxy_port"):
if config.get("proxy_username") and config.get("proxy_password"):
proxy_info = ProxyInfo(proxy_type=proxy_type,
proxy_host=config["proxy_url"],
proxy_port=int(config["proxy_port"]),
proxy_user=config["proxy_username"],
proxy_pass=config["proxy_password"],
proxy_rdns=rdns)
else:
proxy_info = ProxyInfo(proxy_type=proxy_type,
proxy_host=config["proxy_url"],
proxy_port=int(config["proxy_port"]),
proxy_rdns=rdns)
if proxy_info:
http = Http(proxy_info=proxy_info, timeout=timeout,
disable_ssl_certificate_validation=disable_ssl_validation)
else:
http = Http(timeout=timeout,
disable_ssl_certificate_validation=disable_ssl_validation)
if config.get("username") and config.get("password"):
http.add_credentials(config["username"], config["password"])
return http
def process_event(self, *args, **kwargs):
raise NotImplemented()
def pre_handle(self, num, result):
result.setdefault('rid', str(num))
self.update(result)
return result
def get_events(self):
self.result_handle = gzip.open(self.results_file, 'rb')
return (self.pre_handle(num, result) for num, result in enumerate(csv.DictReader(self.result_handle)))
def prepare_meta_for_cam(self):
with gzip.open(self.results_file, 'rb') as rf:
for num, result in enumerate(csv.DictReader(rf)):
result.setdefault('rid', str(num))
self.update(result)
self.invoke()
break
def run(self, argv):
status = 0
if len(argv) < 2 or argv[1] != "--execute":
msg = 'Error: argv="{}", expected="--execute"'.format(argv)
print >> sys.stderr, msg
sys.exit(1)
try:
status = self.process_event()
except Exception as e:
msg = "Unexpected error: {}."
if e.message:
self.log_error(msg.format(e.message))
else:
import traceback
self.log_error(msg.format(traceback.format_exc()))
sys.exit(2)
finally:
if self.result_handle:
self.result_handle.close()
return status
package/bin/da_ess_contentupdate/cim_actions.py
-508
View File
@@ -1,508 +0,0 @@
import collections
import csv
import json
import logging
import logging.handlers
import os
import random
import re
import splunk.rest as rest
import time
from splunk.clilib.bundle_paths import make_splunkhome_path
from splunk.util import mktimegm, normalizeBoolean
# set the maximum allowable CSV field size
#
# The default of the csv module is 128KB; upping to 10MB. See SPL-12117 for
# the background on issues surrounding field sizes.
# (this method is new in python 2.5)
csv.field_size_limit(10485760)
class InvalidResultID(Exception):
pass
class ModularAction(object):
DEFAULT_MSGFIELDS = ['signature',
'action_name',
'search_name',
'sid',
'orig_sid',
'rid',
'orig_rid',
'app',
'user',
'action_mode',
'action_status']
DEFAULT_MESSAGE = 'sendmodaction - ' + ' '.join(['{i}="{{d[{i}]}}"'.format(i=i) for i in DEFAULT_MSGFIELDS])
# The above yields a string.format() compatible format string:
#
# 'sendmodaction - signature="{d[signature]}" action_name="{d[action_name]}"
# search_name="{d[search_name]}" sid="{d[sid]}" orig_sid="{d[orig_sid]}"
# rid="{d[rid]}" orig_rid="{d[orig_rid]}" app="{d[app]}" user="{d[user]}"
# action_mode="{d[action_mode]}" action_status="{d[action_status]}"'
DEFAULT_DROPEXP = lambda x: ((x.startswith('_') and x not in ['_raw', '_time'])
or x.startswith('date_')
or x in ['punct', 'sid', 'rid', 'orig_sid', 'orig_rid'])
DEFAULT_MAPEXP = lambda x: (x.startswith('tag::')
or x in ['_time', '_raw', 'splunk_server', 'index',
'source', 'sourcetype', 'host', 'linecount',
'timestartpos', 'timeendpos', 'eventtype',
'tag', 'search_name', 'event_hash', 'event_id'])
DEFAULT_HEADER = '***SPLUNK*** index="%s" host="%s" source="%s"'
DEFAULT_BREAKER = '==##~~##~~ 1E8N3D4E6V5E7N2T9 ~~##~~##==\n'
DEFAULT_IDLINE = '***Common Action Model*** orig_action_name="%s" orig_sid="%s" orig_rid="%s" sourcetype="%s"\n'
DEFAULT_INDEX = 'summary'
DEFAULT_CHUNK = 50000
SHORT_FORMAT = '%(asctime)s %(levelname)s %(message)s'
def __init__(self, settings, logger, action_name='unknown'):
""" Initialize ModularAction class.
@param settings: A modular action payload in JSON format.
@param logger: A logging instance.
Recommend using ModularAction.setup_logger.
@param action_name: The action name.
action_name in payload will take precedence.
"""
self.settings = json.loads(settings)
self.logger = logger
self.session_key = self.settings.get('session_key')
self.sid = self.settings.get('sid')
self.sid_snapshot = ''
## if sid contains rt_scheduler with snapshot-sid; drop snapshot-sid
## sometimes self.sid may be an integer (1465593470.1228)
try:
rtsid = re.match('^(rt_scheduler.*)\.(\d+)$', self.sid)
if rtsid:
self.sid = rtsid.group(1)
self.sid_snapshot = rtsid.group(2)
except:
pass
## rid_ntuple is a named tuple that represents
## the three variables that change on a per-result basis
self.rid_ntuple = collections.namedtuple('ID', ['orig_sid','rid','orig_rid'])
## rids is a list of rid_ntuple values
## automatically maintained by update() calls
self.rids = []
## current orig_sid based on update()
## aka self.rids[-1].orig_sid
self.orig_sid = ''
## current rid based on update()
## aka self.rids[-1].rid
self.rid = ''
## current orig_rid based on update()
## aka self.rids[-1].orig_rid
self.orig_rid = ''
self.results_file = self.settings.get('results_file')
## info
self.info = {}
if self.results_file:
self.info_file = os.path.join(os.path.dirname(self.results_file), 'info.csv')
self.search_name = self.settings.get('search_name')
self.app = self.settings.get('app')
self.user = self.settings.get('user') or self.settings.get('owner')
self.configuration = self.settings.get('configuration', {})
## enforce configuration is a 'dict'
if not isinstance(self.configuration, dict):
self.configuration = {}
## set loglevel to DEBUG if verbose
if normalizeBoolean(self.configuration.get('verbose', 'false')):
self.logger.setLevel(logging.DEBUG)
self.logger.debug('loglevel set to DEBUG')
## use | sendalert param.action_name=$action_name$
self.action_name = self.configuration.get('action_name') or action_name
## use sid to determine action_mode
if isinstance(self.sid, basestring) and 'scheduler' in self.sid:
self.action_mode = 'saved'
else:
self.action_mode = 'adhoc'
self.action_status = ''
## Since we don't use the result object we get from settings it will be purged
try:
del self.settings['result']
except Exception:
pass
## events
self.events = []
def addinfo(self):
""" The purpose of this method is to populate the
modular action info variable with the contents of info.csv.
@raise Exception: raises Exception if self.info_file could not be opened
or if there were problems parsing the info.csv data
"""
if self.info_file:
try:
with open(self.info_file, 'rU') as fh:
self.info = csv.DictReader(fh).next()
except Exception as e:
self.message('Could not retrieve info.csv', level=logging.WARN)
def addjobinfo(self):
""" The purpose of this method is to populate the job variable
with the contents from REST (/services/search/jobs/<sid>)
SPL-112815 - sendalert - not all $job.<param>$ parameters come through
@raise Exception: raises Exception if search job information could not
be retrieved via REST (search/jobs) based on self.sid
"""
self.job = {}
if self.sid:
try:
response, content = rest.simpleRequest('search/jobs/%s' % self.sid,
sessionKey=self.session_key,
getargs={'output_mode': 'json'})
if response.status == 200:
self.job = json.loads(content)['entry'][0]['content']
self.message('Successfully retrieved search job info')
self.logger.debug(self.job)
else:
self.message('Could not retrieve search job info', level=logging.WARN)
except Exception as e:
self.message('Could not retrieve search job info', level=logging.WARN)
def message(self, signature, status=None, rids=None, level=logging.INFO, **kwargs):
""" The purpose of this method is to provide a common messaging interface.
@param signature: A string representing the message we want to log.
@param status: An optional status that we want to log.
Defaults to None.
@param rids: An optional list of rid_ntuple values in case we
want to generate the message for multiple rids.
Defaults to None (use the rid currently loaded).
@param level: The logging level to use when writing the message.
Defaults to logging.INFO (INFO)
@param kwargs: Additional keyword arguments to be included with the
message.
Defaults to "no arguments".
@return message: This method logs the message; however, for
backwards compatibility we also return the message.
"""
## status
status = status or self.action_status or ''
## rid
if not isinstance(rids, list):
rids = [self.rid_ntuple(self.orig_sid, self.rid, self.orig_rid)]
## kwargs - prune any duplicate keys based on DEFAULT_MSGFIELDS
## prune any keys with special characters [A-Za-z_]+
newargs = [x for x in kwargs\
if (x not in ModularAction.DEFAULT_MSGFIELDS) and re.match('[A-Za-z_]+', x)]
## MSG
msg = '%s %s' % (ModularAction.DEFAULT_MESSAGE, ' '.join(['{i}="{{d[{i}]}}"'.format(i=i) for i in newargs]))
# This will set the default value of any value NOT in the dictionary to the
# empty string.
argsdict = collections.defaultdict(str)
# order is important here - here we update first from kwargs, then from our
# expected arg set.
argsdict.update(kwargs)
argsdict.update({
'signature': signature or '',
'action_name': self.action_name or '',
'search_name': self.search_name or '',
'sid': self.sid or '',
'app': self.app or '',
'user': self.user or '',
'action_mode': self.action_mode or '',
'action_status': status
})
for rid_ntuple in rids:
if len(rid_ntuple)==3:
## Update the arguments dictionary
argsdict.update({
'orig_sid': rid_ntuple.orig_sid or '',
'rid': rid_ntuple.rid or '',
'orig_rid': rid_ntuple.orig_rid or ''
})
## This is where the magic happens. The format string will use the
## attributes of "argsdict"
message = msg.format(d=argsdict)
## prune empty string key-value pairs
for match in re.finditer('[A-Za-z_]+=\"\"(\s|$)', message):
message = message.replace(match.group(0),'',1)
message = message.strip()
self.logger.log(level, message)
else:
self.logger.warn('Could not unpack rid_ntuple')
message = ''
return message
def update(self, result):
""" The purpose of this method is to update the ModularAction instance
identifiers based on the current result being operated on.
This is the most important method in the library as it sets up
rid, orig_sid, and orig_rid to be used by subsequent class methods.
Not calling update() immediately for each result before doing additional
work can have adverse affects.
@param signature: A string representing the message we want to log.
@param status: An optional status that we want to log.
Defaults to None.
@param rids: An optional list of rid_ntuple values in case we
want to generate the message for multiple rids.
Defaults to None (use the rid currently loaded).
@param level: The logging level to use when writing the message.
Defaults to logging.INFO (INFO)
@param kwargs: Additional keyword arguments to be included with the
message.
Defaults to "no arguments".
@return message: This method logs the message; however, for
backwards compatiblity we also return the message.
"""
## This is for events/results that were created as the result of a previous action
self.orig_sid = result.get('orig_sid', '')
## This is for events/results that were created as the result of a previous action
self.orig_rid = result.get('orig_rid', '')
if 'rid' in result and isinstance(result['rid'], (basestring, int)):
self.rid = str(result['rid'])
if self.sid_snapshot:
self.rid = '%s.%s' % (self.rid, self.sid_snapshot)
## add result info to list of named tuples
self.rids.append(self.rid_ntuple(self.orig_sid, self.rid, self.orig_rid))
else:
raise InvalidResultID('Result must have an ID')
def invoke(self):
""" The purpose of this method is to generate per-result invocation messages.
This method is used to identify that an action is being attempted on a per-result basis.
Remember to call update() prior to invoke() to ensure that the invocation message
reflects the appropriate identifiers.
"""
self.message('Invoking modular action')
def result2stash(self, result, dropexp=DEFAULT_DROPEXP, mapexp=DEFAULT_MAPEXP, addinfo=False):
""" The purpose of this method is to formulate an event in stash format
@param result: The result dictionary to generate a stash event for.
@param dropexp: A lambda expression used to determine whether a field
should be dropped or not.
Defaults to DEFAULT_DROPEXP.
@param mapexp: A lambda expression used to determine whether a field
should be mapped (prepended with "orig_") or not.
Defaults to DEFAULT_MAPEXP.
@param addinfo: Whether or not to add search information to the event.
"info" includes search_now, info_min_time, info_max_time,
and info_search_time fields.
Requires that information was loaded into the ModularAction
instance via addinfo()
@return _raw: Returns a string which represents the result in stash format.
The following example has been broken onto multiple lines for readability:
06/21/2016 10:00:00 -0700,
search_name="Access - Brute Force Access Behavior Detected - Rule",
search_now=0.000, info_min_time=1466528400.000, info_max_time=1466532600.000, info_search_time=1465296264.179,
key1=key1val, key2=key2val, key3=key3val, key4=key4val1, key4=key4val2, ...
"""
dropexp = dropexp or (lambda x: False)
mapexp = mapexp or (lambda x: False)
orig_dropexp = lambda x: x.startswith('orig_') and x[5:] in result and mapexp(x[5:])
## addinfo
if addinfo:
result['info_min_time'] = self.info.get('_search_et', '0.000')
info_max_time = self.info.get('_search_lt')
if not info_max_time or info_max_time==0 or info_max_time=='0':
info_max_time = '+Infinity'
result['info_max_time'] = info_max_time
result['info_search_time'] = self.info.get('_timestamp', '')
## construct _raw
_raw = '%s' % result.get('_time', mktimegm(time.gmtime()))
if self.search_name:
_raw += ', search_name="%s"' % self.search_name
processed_keys = []
for key, val in sorted(result.items()):
vals = []
## if we have a proper mv field
if (key.startswith('__mv_')
and val and isinstance(val, basestring)
and val.startswith('$') and val.endswith('$')):
real_key = key[5:]
vals = val[1:-1].split('$;$')
## if proper sv field
elif val and not key.startswith('__mv_'):
real_key = key
vals = [val]
## if we have vals and key hasn't been processed
## and key is not to be dropped...
if (vals
and (real_key not in processed_keys)
and not dropexp(real_key)
and not orig_dropexp(real_key)):
## iterate vals
for val in vals:
## format literal '$'
if key.startswith('__mv'):
val = val.replace('$$', '$')
## escape quotes
if isinstance(val, basestring):
val = val.replace('"', r'\"')
## check map
if mapexp(real_key):
_raw += ', %s="%s"' % ('orig_' + real_key.lstrip('_'), val)
else:
_raw += ', %s="%s"' % (real_key, val)
processed_keys.append(real_key)
return _raw
def addevent(self, raw, sourcetype, cam_header=True):
""" The purpose of this method is to add a properly constructed event
to the events list in the ModularAction instance. This ensures events
are created with the appropriate index-time header.
The index-time header is responsible for setting sourcetype,
orig_action_name, orig_sid, and orig_rid. The index-time header will
not be present in the _raw of generated events.
Remember to call update() prior to addevent() to ensure that the events
reflect the appropriate orig_sid and orig_rid identifiers.
@param raw: The text of the event you want to generate.
@param sourcetype: The sourcetype of the event you want to generate.
@param cam_header: Optionally exclude the inclusion of the index-time header.
Defaults to True (include header).
"""
if cam_header:
if self.orig_sid:
action_idline = ModularAction.DEFAULT_IDLINE % (
self.action_name,
self.orig_sid,
self.orig_rid,
sourcetype)
else:
action_idline = ModularAction.DEFAULT_IDLINE % (
self.action_name,
self.sid,
self.rid,
sourcetype)
self.events.append(action_idline + raw)
else:
self.events.append(raw)
def writeevents(self, index='summary', host=None, source=None, fext='common_action_model'):
""" The purpose of this method is to create arbitrary splunk events
from the list of events in the ModularAction instance.
Please use addevent() for populating the list of events in
the ModularAction instance.
@param index: The index to write the events to.
Defaults to "summary".
@param host: The value of host the events should take on.
Defaults to None (auto).
@param source: The value of source the events should take on.
Defaults to None (auto).
@param fext: The extension of the file to write out.
Files are written to $SPLUNK_HOME/var/spool/splunk.
File extensions can only contain word characters,
dash, and have a 200 char max.
"stash_" is automatically prepended to all extensions.
Defaults to "common_action_model" ("stash_common_action_model").
Only override if you've set up a corresponding props.conf
stanza to handle the extension.
@return bool: Returns True if all events were successfully written
Returns False if any errors were encountered
"""
## internal makeevents method for normalizing strings
## that will be used in the various headers we write out
def get_string(input, default):
try:
return input.replace('"', '_')
except AttributeError:
return default
if self.events:
## sanitize file extension
if not fext or not re.match('^[\w-]+$', fext):
self.logger.warn('Requested file extension was ignored due to invalid characters')
fext = 'common_action_model'
elif len(fext)>200:
self.logger.warn('Requested file extension was ignored due to length')
fext = 'common_action_model'
## header
header_line = ModularAction.DEFAULT_HEADER % (
get_string(index, ModularAction.DEFAULT_INDEX),
get_string(host, ''),
get_string(source, ''))
## process event chunks
for chunk in (self.events[x:x+ModularAction.DEFAULT_CHUNK]
for x in xrange(0, len(self.events), ModularAction.DEFAULT_CHUNK)):
## initialize output string
default_breaker = '\n' + ModularAction.DEFAULT_BREAKER
fout = header_line + default_breaker + (default_breaker).join(chunk)
## write output string
try:
fn = '%s_%s.stash_%s' % (mktimegm(time.gmtime()), random.randint(0, 100000), fext)
fp = make_splunkhome_path(['var', 'spool', 'splunk', fn])
## obtain fh
with open(fp, 'w') as fh:
fh.write(fout)
except:
signature = 'Error obtaining file handle during makeevents'
self.message(signature, level=logging.ERROR, file_path=fp)
self.logger.exception(signature + ' file_path=%s' % fp)
return False
self.message('Successfully created splunk events', event_count=len(self.events))
return True
return False
def dowork(self):
""" This method serves as an illustration stub.
Serves as a container for operations which satisfy the nature of the action.
For instance, the third party API call.
For cleanliness it is recommended that you subclass ModularAction
and implement your own dowork() method.
"""
return
@staticmethod
def setup_logger(name, level=logging.INFO, maxBytes=25000000, backupCount=5, format=SHORT_FORMAT):
""" Set up a logging instance.
@param name: The log file name.
We recommend "$action_name$_modalert".
@param level: The logging level.
@param maxBytes: The maximum log file size before rollover.
@param backupCount: The number of log files to retain.
@return logger: Returns an instance of logger
"""
logfile = make_splunkhome_path(['var', 'log', 'splunk', name + '.log'])
logger = logging.getLogger(name)
logger.setLevel(level)
logger.propagate = False # Prevent the log messages from being duplicated in the python.log file
# Prevent re-adding handlers to the logger object, which can cause duplicate log lines.
handler_exists = any([True for h in logger.handlers if h.baseFilename == logfile])
if not handler_exists:
file_handler = logging.handlers.RotatingFileHandler(logfile, maxBytes=maxBytes, backupCount=backupCount)
formatter = logging.Formatter(format)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
return logger
package/bin/da_ess_contentupdate/cloudconnectlib/__init__.py
-10
View File
@@ -1,10 +0,0 @@
"""
APP Cloud Connect
"""
import os
from .common.lib_util import register_cacert_locater
register_cacert_locater(os.path.join(os.path.dirname(__file__), 'core', 'cacerts'))
__version__ = '1.0.1'
package/bin/da_ess_contentupdate/cloudconnectlib/client.py
-77
View File
@@ -1,77 +0,0 @@
import copy
import os.path
import traceback
from .common.log import get_cc_logger
from .common.util import load_json_file
from .configuration import get_loader_by_version
from .core import CloudConnectEngine
from .core.exceptions import ConfigException
_logger = get_cc_logger()
class CloudConnectClient(object):
"""The client of cloud connect used to start a cloud connect engine instance.
"""
def __init__(self, context, config_file, checkpoint_mgr):
"""
Constructs a `CloudConnectClient` with `context` which contains variables
to render template in the configuration parsed from file `config_file`.
:param context: context to render template.
:param config_file: file path for load user passed interface.
"""
self._context = context
self._config_file = config_file
self._engine = None
self._config = None
self._checkpoint_mgr = checkpoint_mgr
def _load_config(self):
"""Load a JSON based configuration definition from file.
:return: A `dict` contains user defined JSON interface.
"""
try:
conf = load_json_file(self._config_file)
except:
raise ConfigException(
'Unable to load configuration file %s: %s'
% (self._config_file, traceback.format_exc())
)
version = conf.get('meta', {'apiVersion', None}).get('apiVersion', None)
if not version:
raise ConfigException(
'Config meta or api version not present in {}'.format(
self._config_file))
config_loader, schema_file = get_loader_by_version(version)
schema_path = os.path.join(
os.path.dirname(__file__), 'configuration', schema_file)
return config_loader.load(conf, schema_path, self._context)
def start(self):
"""
Initialize a new `CloudConnectEngine` instance and start it.
"""
try:
if self._config is None:
self._config = self._load_config()
self._engine = CloudConnectEngine()
self._engine.start(
context=copy.deepcopy(self._context),
config=self._config,
checkpoint_mgr=self._checkpoint_mgr
)
except Exception as ex:
_logger.exception('Error while starting client')
raise ex
def stop(self):
"""Stop the current cloud connect engine.
"""
if self._engine:
self._engine.stop()
package/bin/da_ess_contentupdate/cloudconnectlib/common/__init__.py
View File
package/bin/da_ess_contentupdate/cloudconnectlib/common/lib_util.py
-54
View File
@@ -1,54 +0,0 @@
import os
import os.path as op
import platform
import sys
import __main__
from ..splunktacollectorlib.common import log as stulog
def get_main_file():
"""Return the running mod input file"""
return __main__.__file__
def get_app_root_dir():
"""Return the root dir of app"""
return op.dirname(op.dirname(op.abspath(get_main_file())))
def get_mod_input_script_name():
"""Return the name of running mod input"""
script_name = os.path.basename(get_main_file())
if script_name.lower().endswith('.py'):
script_name = script_name[:-3]
return script_name
def register_module(new_path):
""" register_module(new_path): adds a directory to sys.path.
Do nothing if it does not exist or if it's already in sys.path.
"""
if not os.path.exists(new_path):
return
new_path = os.path.abspath(new_path)
if platform.system() == 'Windows':
new_path = new_path.lower()
for x in sys.path:
x = os.path.abspath(x)
if platform.system() == 'Windows':
x = x.lower()
if new_path in (x, x + os.sep):
return
sys.path.insert(0, new_path)
def register_cacert_locater(cacerts_locater_path):
for x in sys.modules:
if (x == "httplib2" or x.endswith(".httplib2")) and sys.modules[x] \
is not None:
stulog.logger.warning("Httplib2 module '{}' is already installed. "
"The ca_certs_locater may not work".format(x))
register_module(cacerts_locater_path)
package/bin/da_ess_contentupdate/cloudconnectlib/common/log.py
-32
View File
@@ -1,32 +0,0 @@
import logging
from solnlib.pattern import Singleton
from ..splunktacollectorlib.common import log as stulog
class CloudClientLogAdapter(logging.LoggerAdapter):
__metaclass__ = Singleton
def __init__(self, logger=None, extra=None, prefix=""):
super(CloudClientLogAdapter, self).__init__(logger, extra)
self.cc_prefix = prefix if prefix else ""
def process(self, msg, kwargs):
msg = "{} {}".format(self.cc_prefix, msg)
return super(CloudClientLogAdapter, self).process(msg, kwargs)
def set_level(self, val):
self.logger.setLevel(val)
_adapter = CloudClientLogAdapter(stulog.logger)
def set_cc_logger(logger, logger_prefix=''):
global _adapter
_adapter.logger = logger
_adapter.cc_prefix = logger_prefix or ''
def get_cc_logger():
return _adapter
package/bin/da_ess_contentupdate/cloudconnectlib/common/util.py
-48
View File
@@ -1,48 +0,0 @@
import json
from ..splunktalib.common import util
from solnlib.modular_input.event import XMLEvent
def is_valid_bool(val):
"""Check whether a string can be convert to bool.
:param val: value as string.
:return: `True` if value can be convert to bool else `False`.
"""
return util.is_true(val) or util.is_false(val)
def is_true(val):
return util.is_true(val)
def is_valid_port(port):
"""Check whether a port is valid.
:param port: port to check.
:return: `True` if port is valid else `False`.
"""
try:
return 1 <= int(port) <= 65535
except ValueError:
return False
def load_json_file(file_path):
"""
Load a dict from a JSON file.
:param file_path: JSON file path.
:return: A `dict` object.
"""
with open(file_path, 'r') as file_pointer:
return json.load(file_pointer)
def format_events(raw_events, time=None,
index=None, host=None, source=None, sourcetype=None,
stanza=None, unbroken=False, done=False):
return XMLEvent.format_events(XMLEvent(data, time=time,
index=index, host=host,
source=source,
sourcetype=sourcetype,
stanza=stanza, unbroken=unbroken,
done=done) for data in
raw_events)
package/bin/da_ess_contentupdate/cloudconnectlib/configuration/__init__.py
-1
View File
@@ -1 +0,0 @@
from .loader import get_loader_by_version
package/bin/da_ess_contentupdate/cloudconnectlib/configuration/loader.py
-300
View File
@@ -1,300 +0,0 @@
import logging
import re
import traceback
from abc import abstractmethod
from jsonschema import validate, ValidationError
from munch import munchify
from ..common.log import get_cc_logger
from ..common.util import (
load_json_file, is_valid_bool, is_valid_port, is_true
)
from ..core.exceptions import ConfigException
from ..core.ext import lookup_method
from ..core.models import (
BasicAuthorization, Request, Processor,
Condition, Task, Checkpoint, IterationMode,
DictToken
)
_logger = get_cc_logger()
_PROXY_TYPES = ['http', 'socks4', 'socks5', 'http_no_tunnel']
_AUTH_TYPES = {
'basic_auth': BasicAuthorization
}
_LOGGING_LEVELS = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'FATAL': logging.FATAL,
'CRITICAL': logging.CRITICAL
}
# FIXME Make this configurable
_DEFAULT_LOG_LEVEL = 'INFO'
class CloudConnectConfigLoader(object):
"""The Base cloud connect configuration loader"""
@staticmethod
def _get_schema_from_file(schema_file):
""" Load JSON based schema definition from schema file path.
:return: A `dict` contains schema.
"""
try:
return load_json_file(schema_file)
except:
raise ConfigException(
'Cannot load schema from file {}: {}'.format(
schema_file, traceback.format_exc())
)
@abstractmethod
def load(self, definition, schema_file, context):
pass
class CloudConnectConfigLoaderV1(CloudConnectConfigLoader):
@staticmethod
def _render_from_dict(source, ctx):
rendered = DictToken(source).render(ctx)
return dict((k, v.strip() if isinstance(v, basestring) else v)
for k, v in rendered.iteritems())
def _load_proxy(self, candidate, variables):
"""
Render and validate proxy setting with given variables.
:param candidate: raw proxy setting as `dict`
:param variables: variables to render template in proxy setting.
:return: A `dict` contains rendered proxy setting.
"""
if not candidate:
return {}
proxy = self._render_from_dict(candidate, variables)
enabled = proxy.get('enabled', '0')
if not is_valid_bool(enabled):
raise ValueError(
'Proxy "enabled" expect to be bool type: {}'.format(enabled)
)
proxy['enabled'] = is_true(enabled)
host, port = proxy.get('host'), proxy.get('port')
if host or port:
if not host:
raise ValueError('Proxy "host" must not be empty')
if not is_valid_port(port):
raise ValueError(
'Proxy "port" expect to be in range [1,65535]: %s' % port
)
# proxy type default to 'http'
proxy_type = proxy.get('type')
proxy_type = proxy_type.lower() if proxy_type else 'http'
if proxy_type not in _PROXY_TYPES:
raise ValueError(
'Proxy "type" expect to be one of [{}]: {}'.format(
','.join(_PROXY_TYPES), proxy_type)
)
else:
proxy['type'] = proxy_type
# proxy rdns default to '0'
proxy_rdns = proxy.get('rdns', '0')
if not is_valid_bool(proxy_rdns):
raise ValueError(
'Proxy "rdns" expect to be bool type: {}'.format(proxy_rdns)
)
else:
proxy['rdns'] = is_true(proxy_rdns)
return proxy
@staticmethod
def _get_log_level(level_name):
if level_name:
level_name = level_name.upper().strip()
for k, v in _LOGGING_LEVELS.iteritems():
if k.startswith(level_name):
return v
_logger.warning(
'The log level "%s" is invalid, set it to default: "%s"',
level_name, _DEFAULT_LOG_LEVEL
)
return _LOGGING_LEVELS[_DEFAULT_LOG_LEVEL]
def _load_logging(self, log_setting, variables):
logger = self._render_from_dict(log_setting, variables)
logger['level'] = self._get_log_level(logger.get('level'))
return logger
def _load_global_setting(self, candidate, variables):
"""
Load and render global setting with variables.
:param candidate: Global setting as a `dict`
:param variables: variables from context to render setting
:return: A `Munch` object
"""
candidate = candidate or {}
proxy_setting = self._load_proxy(candidate.get('proxy'), variables)
log_setting = self._load_logging(candidate.get('logging'), variables)
return munchify({'proxy': proxy_setting, 'logging': log_setting})
@staticmethod
def _load_authorization(candidate):
if candidate is None:
return None
auth_type = candidate['type'].lower()
if auth_type not in _AUTH_TYPES:
raise ValueError(
'Auth type expect to be one of [{}]: {}'.format(
','.join(_AUTH_TYPES.keys()), auth_type)
)
return _AUTH_TYPES[auth_type](candidate['options'])
def _load_options(self, options):
return Request(
auth=self._load_authorization(options.get('auth')),
url=options['url'],
method=options.get('method', 'GET'),
header=options.get('headers', {}),
body=options.get('body', {})
)
@staticmethod
def _validate_method(method):
if lookup_method(method) is None:
raise ValueError('Unimplemented method: {}'.format(method))
def _parse_tasks(self, raw_tasks):
tasks = []
for item in raw_tasks:
self._validate_method(item['method'])
tasks.append(Task(item['input'], item['method'], item.get('output')))
return tasks
def _parse_conditions(self, raw_conditions):
conditions = []
for item in raw_conditions:
self._validate_method(item['method'])
conditions.append(Condition(item['input'], item['method']))
return conditions
@staticmethod
def _load_checkpoint(checkpoint):
if not checkpoint:
return None
return Checkpoint(
checkpoint.get('namespace', []), checkpoint['content'])
def _load_iteration_mode(self, iteration_mode):
count = iteration_mode.get('iteration_count', '0')
try:
iteration_count = int(count)
except ValueError:
raise ValueError(
'"iteration_count" must be an integer: %s' % count)
stop_conditions = self._parse_conditions(
iteration_mode['stop_conditions'])
return IterationMode(iteration_count=iteration_count,
conditions=stop_conditions)
def _load_processor(self, processor):
skip_conditions = self._parse_conditions(
processor.get('skip_conditions', [])
)
pipeline = self._parse_tasks(processor.get('pipeline', []))
return Processor(
skip_conditions=skip_conditions,
pipeline=pipeline
)
def _load_request(self, request):
options = self._load_options(request['request'])
pre_process = self._load_processor(request.get('pre_process', {}))
post_process = self._load_processor(request['post_process'])
checkpoint = self._load_checkpoint(request.get('checkpoint'))
iteration_mode = self._load_iteration_mode(request['iteration_mode'])
return munchify({
'request': options,
'pre_process': pre_process,
'post_process': post_process,
'checkpoint': checkpoint,
'iteration_mode': iteration_mode,
})
def load(self, definition, schema_file, context):
"""Load cloud connect configuration from a `dict` and validate
it with schema and global settings will be rendered.
:param schema_file: Schema file location used to validate config.
:param definition: A dictionary contains raw configs.
:param context: variables to render template in global setting.
:return: A `Munch` object.
"""
try:
validate(definition, self._get_schema_from_file(schema_file))
except ValidationError:
raise ConfigException(
'Failed to validate interface with schema: {}'.format(
traceback.format_exc()))
try:
global_settings = self._load_global_setting(
definition.get('global_settings'), context
)
requests = [self._load_request(item) for item in definition['requests']]
return munchify({
'meta': munchify(definition['meta']),
'tokens': definition['tokens'],
'global_settings': global_settings,
'requests': requests,
})
except Exception as ex:
error = 'Unable to load configuration: %s' % str(ex)
_logger.exception(error)
raise ConfigException(error)
_loader_and_schema_by_version = {
r'1\.0\.0': (CloudConnectConfigLoaderV1, 'schema_1_0_0.json'),
}
def get_loader_by_version(version):
""" Instantiate a configuration loader on basis of a given version.
A `ConfigException` will raised if the version is not supported.
:param version: Version to lookup config loader.
:return: A config loader.
"""
for support_version in _loader_and_schema_by_version:
if re.match(support_version, version):
loader_cls, schema = _loader_and_schema_by_version[support_version]
return loader_cls(), schema
raise ConfigException(
'Unsupported schema version {}, current supported'
' versions should match these regex [{}]'.format(version, ','.join(
_loader_and_schema_by_version))
)
package/bin/da_ess_contentupdate/cloudconnectlib/configuration/schema_1_0_0.json
-344
View File
@@ -1,344 +0,0 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"additionalProperties": false,
"definitions": {
"Authentication": {
"additionalProperties": false,
"properties": {
"options": {
"type": "object"
},
"type": {
"enum": [
"digest",
"basic_auth"
],
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
"Checkpoint": {
"additionalProperties": false,
"properties": {
"content": {
"type": "object"
},
"namespace": {
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"content"
],
"type": "object"
},
"GlobalSettings": {
"additionalProperties": false,
"properties": {
"logging": {
"properties": {
"level": {
"type": "string"
}
},
"type": "object"
},
"proxy": {
"$ref": "#/definitions/Proxy"
}
},
"type": "object"
},
"IterationMode": {
"additionalProperties": false,
"properties": {
"iteration_count": {
"oneOf": [
{
"pattern": "^[+-]?[1-9]\\d*|0$",
"type": "string"
},
{
"type": "integer"
}
]
},
"stop_conditions": {
"items": {
"additionalProperties": false,
"properties": {
"input": {
"items": {
"type": "string"
},
"type": "array"
},
"method": {
"type": "string"
}
},
"required": [
"input",
"method"
],
"type": "object"
},
"type": "array"
}
},
"type": "object"
},
"Meta": {
"additionalProperties": false,
"properties": {
"apiVersion": {
"pattern": "(?:\\d{1,3}\\.){2}[\\w\\-]{1,15}",
"type": "string"
}
},
"required": [
"apiVersion"
],
"type": "object"
},
"Proxy": {
"additionalProperties": false,
"properties": {
"enabled": {
"default": false,
"oneOf": [
{
"type": "string"
},
{
"type": "boolean"
},
{
"type": "integer"
}
]
},
"host": {
"type": "string"
},
"password": {
"type": "string"
},
"port": {
"oneOf": [
{
"type": "string"
},
{
"exclusiveMaximum": true,
"exclusiveMinimum": true,
"maximum": 65535,
"minimum": 1,
"type": "integer"
}
]
},
"rdns": {
"type": "string"
},
"type": {
"type": "string"
},
"username": {
"type": "string"
}
},
"required": [
"host",
"port"
],
"type": "object"
},
"Request": {
"additionalProperties": false,
"properties": {
"checkpoint": {
"$ref": "#/definitions/Checkpoint"
},
"iteration_mode": {
"$ref": "#/definitions/IterationMode"
},
"post_process": {
"additionalProperties": false,
"properties": {
"pipeline": {
"items": {
"additionalProperties": false,
"properties": {
"input": {
"items": {
"type": "string"
},
"type": "array"
},
"method": {
"type": "string"
},
"output": {
"type": "string"
}
},
"required": [
"input",
"method"
],
"type": "object"
},
"type": "array"
},
"skip_conditions": {
"items": {
"additionalProperties": false,
"properties": {
"input": {
"items": {
"type": "string"
},
"type": "array"
},
"method": {
"type": "string"
}
},
"required": [
"input",
"method"
],
"type": "object"
},
"type": "array"
}
},
"type": "object"
},
"pre_process": {
"additionalProperties": false,
"properties": {
"pipeline": {
"items": {
"additionalProperties": false,
"properties": {
"input": {
"items": {
"type": "string"
},
"type": "array"
},
"method": {
"type": "string"
},
"output": {
"type": "string"
}
},
"required": [
"input",
"method"
],
"type": "object"
},
"type": "array"
},
"skip_conditions": {
"items": {
"additionalProperties": false,
"properties": {
"input": {
"items": {
"type": "string"
},
"type": "array"
},
"method": {
"type": "string"
}
},
"required": [
"input",
"method"
],
"type": "object"
},
"type": "array"
}
},
"type": "object"
},
"request": {
"additionalProperties": false,
"properties": {
"auth": {
"$ref": "#/definitions/Authentication"
},
"body": {
"type": "object"
},
"headers": {
"type": "object"
},
"method": {
"default": "GET",
"enum": [
"GET",
"POST"
],
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"url"
],
"type": "object"
}
},
"required": [
"request",
"post_process",
"iteration_mode"
],
"type": "object"
}
},
"properties": {
"global_settings": {
"$ref": "#/definitions/GlobalSettings"
},
"meta": {
"$ref": "#/definitions/Meta"
},
"requests": {
"items": {
"$ref": "#/definitions/Request"
},
"minItems": 1,
"type": "array"
},
"tokens": {
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"meta",
"tokens",
"requests"
],
"type": "object"
}
package/bin/da_ess_contentupdate/cloudconnectlib/core/__init__.py
-2
View File
@@ -1,2 +0,0 @@
from .engine import CloudConnectEngine
from .exceptions import ConfigException, HTTPError
package/bin/da_ess_contentupdate/cloudconnectlib/core/cacerts/ca_certs_locater.py
-130
View File
@@ -1,130 +0,0 @@
"""
`ca_certs_locater` is a lib for extending httplib2 to allow system certificate store to be used when
verifying SSL certificates, to enable this lib, you should add it to your python import path before
initializing httplib2. As we're not trying to implement SSL certificate RFCs, parsing and validating
certificates are not included.
"""
import atexit
import os
import os.path as op
import ssl
import sys
TEMP_CERT_FILE_NAME = 'httplib2_merged_certificates_{}.crt'
LINUX_CERT_PATH_1 = '/etc/pki/tls/certs/ca-bundle.crt' # RedHat
LINUX_CERT_PATH_2 = '/etc/ssl/certs/ca-certificates.crt' # Debian
DARWIN_CERT_PATH = '/usr/local/etc/openssl/cert.pem'
HTTPLIB2_CA_CERT_FILE_NAME = 'cacerts.txt'
TEMP_CERT_FILE_PATH = None
def get():
"""
Returns: a path to generated certificate authority file
"""
try:
return _get()
except (IOError, OSError, ssl.SSLError):
_fallback() # IO and SSL relative errors should be swallowed to protect the HTTP request
def _get():
global TEMP_CERT_FILE_PATH
# also check file existence as it's possible for the temp file to be deleted
if TEMP_CERT_FILE_PATH is None or not os.path.exists(TEMP_CERT_FILE_PATH):
temp_cert_file_path = _generate_temp_cert_file_name()
ssl_ca_certs = _read_ssl_default_ca_certs()
if not ssl_ca_certs:
# it's possible the ca load path is not well configured, try some typical paths
ssl_ca_certs = _read_platform_pem_cert_file()
if ssl_ca_certs: # only update temp cert file when there's additional PEM certs found
cert_files = [ssl_ca_certs, _read_httplib2_default_certs()]
_update_temp_cert_file(temp_cert_file_path, cert_files)
TEMP_CERT_FILE_PATH = temp_cert_file_path
else:
_fallback()
return TEMP_CERT_FILE_PATH
def _fallback():
"""
Give up the loading process by throwing specified exception, httplib2 will then use its
bundled certificates
"""
raise ImportError('Unable to load system certificate authority files')
def _read_platform_pem_cert_file():
if sys.platform.startswith('linux'):
pem_files = [_read_pem_file(LINUX_CERT_PATH_1), _read_pem_file(LINUX_CERT_PATH_2)]
return '\n'.join(filter(None, pem_files))
elif sys.platform.startswith('darwin'):
return _read_pem_file(DARWIN_CERT_PATH)
else:
return ""
def _read_ssl_default_ca_certs():
# it's not guaranteed to return PEM formatted certs when `binary_form` is False
der_certs = ssl.create_default_context().get_ca_certs(binary_form=True)
pem_certs = [ssl.DER_cert_to_PEM_cert(der_cert_bytes) for der_cert_bytes in der_certs]
return '\n'.join(pem_certs)
def _read_httplib2_default_certs():
import httplib2 # import error should not happen here, and will be well handled by outer called
httplib_dir = os.path.dirname(os.path.abspath(httplib2.__file__))
ca_certs_path = os.path.join(httplib_dir, HTTPLIB2_CA_CERT_FILE_NAME)
return _read_pem_file(ca_certs_path)
def _read_pem_file(path):
if os.path.exists(path):
with open(path, mode='r') as pem_file:
return pem_file.read()
else:
return ""
def _update_temp_cert_file(temp_file, pem_texts):
with open(temp_file, mode='w') as temp_cert_file:
for pem_text in pem_texts:
if len(pem_text) > 0:
temp_cert_file.write(pem_text + '\n')
temp_cert_file.flush()
atexit.register(_do_safe_remove, temp_file)
def _do_safe_remove(file_path):
if os.path.exists(file_path):
try:
os.remove(file_path)
except:
pass
def _get_temp_cert_file_dir():
import __main__
app_root = op.dirname(op.dirname(op.abspath(__main__.__file__)))
temp_dir = op.join(app_root, 'temp_certs')
if not op.isdir(temp_dir):
try:
os.mkdir(temp_dir)
except:
pass
for candidate in ['temp_certs', 'local', 'default']:
dir_path = op.join(app_root, candidate)
if op.isdir(dir_path):
return dir_path
return app_root
def _generate_temp_cert_file_name():
file_name = TEMP_CERT_FILE_NAME.format(os.getpid())
return os.path.join(_get_temp_cert_file_dir(), file_name)
package/bin/da_ess_contentupdate/cloudconnectlib/core/defaults.py
-20
View File
@@ -1,20 +0,0 @@
"""Default config for cloud connect"""
timeout = 120 # request timeout is two minutes
disable_ssl_cert_validation = False # default enable SSL validation
success_statuses = (200, 201) # statuses be treated as success.
# response status which need to retry.
retry_statuses = (429, 500, 501, 502, 503, 504, 505, 506, 507,
509, 510, 511)
# response status which need print a warning log.
warning_statuses = (203, 204, 205, 206, 207, 208, 226,
300, 301, 302, 303, 304, 305, 306, 307, 308)
retries = 3 # Default maximum retry times.
max_iteration_count = 100 # maximum iteration loop count
charset = 'utf-8' # Default response charset if not found in response header
package/bin/da_ess_contentupdate/cloudconnectlib/core/engine.py
-326
View File
@@ -1,326 +0,0 @@
import json
import threading
from . import defaults
from .exceptions import HTTPError, StopCCEIteration
from .http import HTTPRequest
from ..common.log import get_cc_logger
_logger = get_cc_logger()
class CloudConnectEngine(object):
"""The cloud connect engine to process request instantiated
from user options."""
def __init__(self):
self._stopped = False
self._running_job = None
@staticmethod
def _set_logging(log_setting):
_logger.set_level(log_setting.level)
def start(self, context, config, checkpoint_mgr):
"""Start current client instance to execute each request parsed
from config.
"""
if not config:
raise ValueError('Config must not be empty')
context = context or {}
global_setting = config.global_settings
CloudConnectEngine._set_logging(global_setting.logging)
_logger.info('Start to execute requests jobs.')
processed = 0
for request in config.requests:
job = Job(
request=request,
context=context,
checkpoint_mgr=checkpoint_mgr,
proxy=global_setting.proxy,
)
self._running_job = job
job.run()
processed += 1
_logger.info('%s job(s) process finished', processed)
if self._stopped:
_logger.info(
'Engine has been stopped, stopping to execute jobs.')
break
self._stopped = True
_logger.info('Engine executing finished')
def stop(self):
"""Stops engine and running job. Do nothing if engine already
been stopped."""
if self._stopped:
_logger.info('Engine already stopped, do nothing.')
return
_logger.info('Stopping engine')
if self._running_job:
_logger.info('Attempting to stop the running job.')
self._running_job.terminate()
_logger.info('Stopping job finished.')
self._stopped = True
class Job(object):
"""Job class represents a single request to send HTTP request until
reached it's stop condition.
"""
def __init__(self, request, context, checkpoint_mgr, proxy=None):
"""
Constructs a `Job` with properties request, context and a
optional proxy setting.
:param request: A `Request` instance which contains request settings.
:param context: A values set contains initial values for template
variables.
:param proxy: A optional `Proxy` object contains proxy related
settings.
"""
self._request = request
self._context = context
self._checkpoint_mgr = checkpoint_mgr
self._client = HTTPRequest(proxy)
self._stopped = True
self._should_stop = False
self._request_iterated_count = 0
self._iteration_mode = self._request.iteration_mode
self._max_iteration_count = self._get_max_iteration_count()
self._running_thread = None
self._terminated = threading.Event()
def _get_max_iteration_count(self):
mode_max_count = self._iteration_mode.iteration_count
default_max_count = defaults.max_iteration_count
return min(default_max_count, mode_max_count) \
if mode_max_count > 0 else default_max_count
def terminate(self, block=True, timeout=30):
"""Terminate this job, the current thread will blocked util
the job is terminate finished if block is True """
if self.is_stopped():
_logger.info('Job already been stopped.')
return
if self._running_thread == threading.current_thread():
_logger.warning('Job cannot terminate itself.')
return
_logger.info('Stopping job')
self._should_stop = True
if not block:
return
if not self._terminated.wait(timeout):
_logger.warning('Terminating job timeout.')
def _set_context(self, key, value):
self._context[key] = value
def _execute_tasks(self, tasks):
if not tasks:
return
for task in tasks:
if self._check_should_stop():
return
self._context.update(task.execute(self._context))
def _on_pre_process(self):
"""
Execute tasks in pre process one by one if condition satisfied.
"""
pre_processor = self._request.pre_process
if pre_processor.should_skipped(self._context):
_logger.info('Skip pre process condition satisfied, do nothing')
return
tasks = pre_processor.pipeline
_logger.debug(
'Got %s tasks need be executed before process', len(tasks))
self._execute_tasks(tasks)
def _on_post_process(self):
"""
Execute tasks in post process one by one if condition satisfied.
"""
post_processor = self._request.post_process
if post_processor.should_skipped(self._context):
_logger.info('Skip post process condition satisfied, '
'do nothing')
return
tasks = post_processor.pipeline
_logger.debug(
'Got %s tasks need to be executed after process', len(tasks)
)
self._execute_tasks(tasks)
def _update_checkpoint(self):
"""Updates checkpoint based on checkpoint namespace and content."""
checkpoint = self._request.checkpoint
if not checkpoint:
_logger.info('Checkpoint not specified, do not update it.')
return
self._checkpoint_mgr.update_ckpt(
checkpoint.normalize_content(self._context),
namespaces=checkpoint.normalize_namespace(self._context),
)
def _get_checkpoint(self):
checkpoint = self._request.checkpoint
if not checkpoint:
_logger.info('Checkpoint not specified, do not read it.')
return
namespaces = checkpoint.normalize_namespace(self._context)
checkpoint = self._checkpoint_mgr.get_ckpt(namespaces)
if checkpoint:
self._context.update(checkpoint)
def _is_stoppable(self):
"""Check if repeat mode conditions satisfied."""
if self._request_iterated_count >= self._max_iteration_count:
_logger.info(
'Job iteration count is %s, current request count is %s,'
' stop condition satisfied.',
self._max_iteration_count, self._request_iterated_count
)
return True
if self._iteration_mode.passed(self._context):
_logger.info('Job stop condition satisfied.')
return True
return False
def is_stopped(self):
"""Return if this job is stopped."""
return self._stopped
def run(self):
"""Start job and exit util meet stop condition. """
_logger.info('Start to process job')
self._stopped = False
try:
self._running_thread = threading.current_thread()
self._run()
except Exception:
_logger.exception('Error encountered while running job.')
raise
finally:
self._terminated.set()
self._stopped = True
_logger.info('Job processing finished')
def _check_should_stop(self):
if self._should_stop:
_logger.info('Job should been stopped.')
return self._should_stop
def _run(self):
request = self._request.request
method = request.method
authorizer = request.auth
self._get_checkpoint()
while 1:
if self._check_should_stop():
return
try:
self._on_pre_process()
except StopCCEIteration:
_logger.info('Stop iteration command in pre process is received, exit job now.')
return
url = request.normalize_url(self._context)
header = request.normalize_header(self._context)
body = request.normalize_body(self._context)
body_json = json.dumps(body) if body else None
if authorizer:
authorizer(header, self._context)
if self._check_should_stop():
return
response, need_terminate = \
self._send_request(url, method, header, body=body_json)
if need_terminate:
_logger.info('This job need to be terminated.')
break
self._request_iterated_count += 1
self._set_context('__response__', response)
if self._check_should_stop():
return
try:
self._on_post_process()
except StopCCEIteration:
_logger.info('Stop iteration command in post process is received, exit job now.')
return
if self._check_should_stop():
return
self._update_checkpoint()
if self._is_stoppable():
_logger.info('Stop condition reached, exit job now')
break
def _send_request(self, url, method, header, body):
"""Do send request with a simple error handling strategy."""
try:
response = self._client.request(
url, method, headers=header, body=body
)
except HTTPError as error:
_logger.exception(
'HTTPError reason=%s when sending request to '
'url=%s method=%s', error.reason, url, method)
return None, True
status = response.status_code
if status in defaults.success_statuses:
if not (response.body or '').strip():
_logger.info(
'The response body of request which url=%s and'
' method=%s is empty, status=%s.',
url, method, status
)
return None, True
return response, False
error_log = ('The response status=%s for request which url=%s and'
' method=%s.') % (
status, url, method
)
if status in defaults.warning_statuses:
_logger.warning(error_log)
else:
_logger.error(error_log)
return None, True
package/bin/da_ess_contentupdate/cloudconnectlib/core/exceptions.py
-27
View File
@@ -1,27 +0,0 @@
"""APP Cloud Connect errors"""
class ConfigException(Exception):
"""Config exception"""
pass
class FuncException(Exception):
"""Ext function call exception"""
pass
class HTTPError(Exception):
""" HTTPError raised when HTTP request returned a error."""
def __init__(self, reason=None):
"""
Initialize HTTPError with `response` object and `status`.
"""
self.reason = reason
super(HTTPError, self).__init__(reason)
class StopCCEIteration(Exception):
"""Exception to exit from the engine iteration."""
pass
package/bin/da_ess_contentupdate/cloudconnectlib/core/ext.py
-336
View File
@@ -1,336 +0,0 @@
import calendar
import json
import re
import traceback
from datetime import datetime
from jsonpath_rw import parse
from .exceptions import FuncException, StopCCEIteration
from .pipemgr import PipeManager
from ..common import util, log
_logger = log.get_cc_logger()
def regex_match(pattern, source, flags=0):
"""
Determine whether a string is match a regex pattern.
:param pattern: regex pattern
:param source: candidate to match regex
:param flags: flags for regex match
:return: `True` if candidate match pattern else `False`
"""
try:
return re.match(pattern, source, flags) is not None
except Exception:
_logger.warning(
'Unable to match source with pattern=%s, cause=%s',
pattern,
traceback.format_exc()
)
return False
def regex_not_match(pattern, source, flags=0):
"""
Determine whether a string is not match a regex pattern.
:param pattern: regex expression
:param source: candidate to match regex
:param flags: flags for regex match
:return: `True` if candidate not match pattern else `False`
"""
return not regex_match(pattern, source, flags)
def json_path(source, json_path_expr):
""" Extract value from string with JSONPATH expression.
:param json_path_expr: JSONPATH expression
:param source: string to extract value
:return: A `list` contains all values extracted
"""
if not source:
_logger.debug('source to apply JSONPATH is empty, return empty.')
return ''
if isinstance(source, basestring):
_logger.debug(
'source expected is a JSON, not %s. Attempt to'
' convert it to JSON',
type(source)
)
try:
source = json.loads(source)
except Exception as ex:
_logger.warning(
'Unable to load JSON from source: %s. '
'Attempt to apply JSONPATH "%s" on source directly.',
ex.message,
json_path_expr
)
try:
expression = parse(json_path_expr)
results = [match.value for match in expression.find(source)]
_logger.debug(
'Got %s elements extracted with JSONPATH expression "%s"',
len(results), json_path_expr
)
if not results:
return ''
return results[0] or '' if len(results) == 1 else results
except Exception as ex:
_logger.warning(
'Unable to apply JSONPATH expression "%s" on source,'
' message=%s cause=%s',
json_path_expr,
ex.message,
traceback.format_exc()
)
return ''
def splunk_xml(candidates,
time=None,
index=None,
host=None,
source=None,
sourcetype=None):
""" Wrap a event with splunk xml format.
:param candidates: data used to wrap as event
:param time: timestamp which must be empty or a valid float
:param index: index name for event
:param host: host for event
:param source: source for event
:param sourcetype: sourcetype for event
:return: A wrapped event with splunk xml format.
"""
if not isinstance(candidates, (list, tuple)):
candidates = [candidates]
time = time or None
if time:
try:
time = float(time)
except ValueError:
_logger.warning(
'"time" %s is expected to be a float, set "time" to None',
time
)
time = None
return util.format_events(
candidates,
time=time,
index=index,
host=host,
source=source,
sourcetype=sourcetype
)
def std_output(candidates):
""" Output a string to stdout.
:param candidates: List of string to output to stdout or a single string.
"""
if isinstance(candidates, basestring):
candidates = [candidates]
all_str = True
for candidate in candidates:
if all_str and not isinstance(candidate, basestring):
all_str = False
_logger.warning(
'The type of data needs to print is "%s" rather than'
' basestring',
type(candidate)
)
if not PipeManager().write_events(candidate):
raise FuncException('Fail to output data to stdout. The event'
' writer is stopped or encountered exception')
_logger.debug('Writing events to stdout finished.')
return True
def _parse_json(source, json_path_expr=None):
if not source:
_logger.debug('Unable to parse JSON from empty source, return empty.')
return {}
if json_path_expr:
_logger.debug(
'Try to extract JSON from source with JSONPATH expression: %s, ',
json_path_expr
)
source = json_path(source, json_path_expr)
elif isinstance(source, basestring):
source = json.loads(source)
return source
def json_empty(source, json_path_expr=None):
"""Check whether a JSON is empty, return True only if the JSON to
check is a valid JSON and is empty.
:param json_path_expr: A optional JSONPATH expression
:param source: source to extract JSON
:return: `True` if the result JSON is empty
"""
try:
data = _parse_json(source, json_path_expr)
if isinstance(data, (list, tuple)):
return all(len(ele) == 0 for ele in data)
return len(data) == 0
except Exception as ex:
_logger.warning(
'Unable to determine whether source is json_empty, treat it as '
'not json_empty: %s', ex.message
)
return False
def json_not_empty(source, json_path_expr=None):
"""Check if a JSON object is not empty, return True only if the
source is a valid JSON object and the value leading by
json_path_expr is empty.
:param json_path_expr: A optional JSONPATH expression
:param source: source to extract JSON
:return: `True` if the result JSON is not empty
"""
try:
data = _parse_json(source, json_path_expr)
if isinstance(data, (list, tuple)):
return any(len(ele) > 0 for ele in data)
return len(data) > 0
except Exception as ex:
_logger.warning(
'Unable to determine whether source is json_not_empty, '
'treat it as not json_not_empty: %s',
ex.message
)
return False
def set_var(value):
"""Set a variable which name should be specified in `output` with value"""
return value
def _fix_microsecond_format(fmt, micros):
"""
implement %Nf so that user can control the digital number of microsecond.
If number of % is even, don't do replacement.
If N is not in [1-6], don't do replacement.
If time length m is less than N, convert it to 6 digitals and return N
digitals.
"""
micros = str(micros).zfill(6)
def do_replacement(x, micros):
if int(x.group(1)) in range(1, 7) and len(x.group()) % 2:
return x.group().replace('%' + x.group(1) + 'f',
micros[:min(int(x.group(1)), len(micros))])
return x.group()
return re.sub(r'%+([1-6])f', lambda x: do_replacement(x, micros), fmt)
def _fix_timestamp_format(fmt, timestamp):
"""Replace '%s' in time format with timestamp if the number
of '%' before 's' is odd."""
return re.sub(
r'%+s',
(
lambda x:
x.group() if len(x.group()) % 2 else x.group().replace('%s',
timestamp)
),
fmt
)
def time_str2str(date_string, from_format, to_format):
"""Convert a date string with given format to another format. Return
the original date string if it's type is not string or failed to parse or
convert it with format."""
if not isinstance(date_string, basestring):
_logger.warning(
'"date_string" must be a string type, found %s,'
' return the original date_string directly.',
type(date_string)
)
return date_string
try:
dt = datetime.strptime(date_string, from_format)
# Need to pre process '%s' in to_format here because '%s' is not
# available on all platforms. Even on supported platforms, the
# result may be different because it depends on implementation on each
# platform. Replace it with UTC timestamp here directly.
if to_format:
timestamp = calendar.timegm(dt.timetuple())
to_format = _fix_timestamp_format(to_format, str(timestamp))
to_format = _fix_microsecond_format(to_format, str(dt.microsecond))
return dt.strftime(to_format)
except Exception:
_logger.warning(
'Unable to convert date_string "%s" from format "%s" to "%s",'
' return the original date_string, cause=%s',
date_string,
from_format,
to_format,
traceback.format_exc()
)
return date_string
def is_true(value):
"""Determine whether value is True"""
return str(value).strip().lower() == 'true'
def exit_if_true(value):
"""Raise a StopCCEIteration exception if value is True"""
if is_true(value):
raise StopCCEIteration
def assert_true(value, message=None):
"""Assert value is True"""
if not is_true(value):
raise AssertionError(
message or '"{value}" is not true'.format(value=value)
)
_extension_functions = {
'assert_true': assert_true,
'exit_if_true': exit_if_true,
'is_true': is_true,
'regex_match': regex_match,
'regex_not_match': regex_not_match,
'set_var': set_var,
'splunk_xml': splunk_xml,
'std_output': std_output,
'json_path': json_path,
'json_empty': json_empty,
'json_not_empty': json_not_empty,
'time_str2str': time_str2str,
}
def lookup_method(name):
""" Find a predefined function with given function name.
:param name: function name.
:return: A function with given name.
"""
return _extension_functions.get(name)
package/bin/da_ess_contentupdate/cloudconnectlib/core/http.py
-234
View File
@@ -1,234 +0,0 @@
import time
import traceback
from httplib2 import ProxyInfo, Http, socks, SSLHandshakeError
from solnlib.packages.requests import PreparedRequest, utils
from . import defaults
from .exceptions import HTTPError
from ..common.log import get_cc_logger
_logger = get_cc_logger()
class HTTPResponse(object):
"""
HTTPResponse class wraps response of HTTP request for later use.
"""
def __init__(self, response, content):
"""Construct a HTTPResponse from response and content returned
with httplib2 request"""
self._status_code = response.status
self._header = response
self._body = self._decode_content(response, content)
@staticmethod
def _decode_content(response, content):
if not content:
return ''
charset = utils.get_encoding_from_headers(response)
if charset is None:
charset = defaults.charset
_logger.info(
'Unable to find charset in response headers,'
' set it to default "%s"', charset
)
_logger.info('Decoding response content with charset=%s', charset)
try:
return content.decode(charset, errors='replace')
except Exception as ex:
_logger.warning(
'Failure decoding response content with charset=%s,'
' decode it with utf-8: %s',
charset, ex.message
)
return content.decode('utf-8', errors='replace')
@property
def header(self):
return self._header
@property
def body(self):
"""
Return response body as a `string`.
:return: A `string`
"""
return self._body
@property
def status_code(self):
"""
Return response status code.
:return: A `integer`
"""
return self._status_code
def _make_prepare_url_func():
"""Expose prepare_url in `PreparedRequest`"""
pr = PreparedRequest()
def prepare_url(url, params=None):
"""Prepare the given HTTP URL with ability provided in requests lib.
For some illegal characters in URL or parameters like space(' ') will
be escaped to make sure we can request the correct URL."""
pr.prepare_url(url, params=params)
return pr.url
return prepare_url
class HTTPRequest(object):
"""
HTTPRequest class represents a single request to send HTTP request until
reached it's stop condition.
"""
_PROXY_TYPE = {
'http': socks.PROXY_TYPE_HTTP,
'http_no_tunnel': socks.PROXY_TYPE_HTTP_NO_TUNNEL,
'socks4': socks.PROXY_TYPE_SOCKS4,
'socks5': socks.PROXY_TYPE_SOCKS5,
}
def __init__(self, proxy=None):
"""Constructs a `HTTPRequest` with a optional proxy setting.
:param proxy: A optional `Proxy` object contains proxy related
settings.
"""
self._proxy_info = self._prepare_proxy_info(proxy)
self._connection = None
self._prepare_url_func = _make_prepare_url_func()
def _send_request(self, uri, method, headers=None, body=None):
"""Do send request to target URL and validate SSL cert by default.
If validation failed, disable it and try again."""
if self._connection is None:
self._connection = self._build_http_connection(
proxy_info=self._proxy_info,
disable_ssl_cert_validation=False)
try:
return self._connection.request(
uri, body=body, method=method, headers=headers
)
except SSLHandshakeError:
_logger.warning(
"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verification failed. "
"The certificate of the https server [%s] is not trusted, "
"this add-on will proceed to connect with this certificate. "
"You may need to check the certificate and "
"refer to the documentation and add it to the trust list. %s",
uri,
traceback.format_exc()
)
self._connection = self._build_http_connection(
proxy_info=self._proxy_info,
disable_ssl_cert_validation=True
)
return self._connection.request(
uri, body=body, method=method, headers=headers
)
def request(self, url, method='GET', headers=None, body=None):
"""
Invoke a request with httplib2 and return it's response.
:param url: url address to send request to.
:param method: request method `GET` by default.
:param headers: request headers.
:param body: request body.
:return: A `HTTPResponse` object.
"""
if body and not isinstance(body, str):
raise TypeError('Request body type must be str')
if self._connection is None:
self._connection = self._build_http_connection(self._proxy_info)
try:
uri = self._prepare_url_func(url)
except Exception:
_logger.warning(
'Failed to encode url=%s: %s, use original url directly',
url, traceback.format_exc()
)
uri = url
_logger.info('Preparing to invoke request to [%s]', uri)
result = self._do_request(uri, method, headers, body)
_logger.info('Invoking request to [%s] finished', uri)
return result
def _prepare_proxy_info(self, proxy):
if not proxy or not proxy.enabled:
_logger.debug('Proxy is not enabled')
return None
username = proxy.username \
if 'username' in proxy and proxy.username else None
password = proxy.password \
if 'password' in proxy and proxy.password else None
proxy_type = self._PROXY_TYPE.get(proxy.type) or self._PROXY_TYPE['http']
return ProxyInfo(proxy_host=proxy.host,
proxy_port=int(proxy.port),
proxy_type=proxy_type,
proxy_user=username,
proxy_pass=password,
proxy_rdns=proxy.rdns)
@staticmethod
def _build_http_connection(
proxy_info=None,
timeout=defaults.timeout,
disable_ssl_cert_validation=defaults.disable_ssl_cert_validation):
return Http(
proxy_info=proxy_info,
timeout=timeout,
disable_ssl_certificate_validation=disable_ssl_cert_validation)
@staticmethod
def _is_need_retry(status, retried, maximum_retries):
return retried < maximum_retries \
and status in defaults.retry_statuses
def _do_request(self, uri, method='GET', headers=None, body=None):
"""Invokes request and auto retry with an exponential backoff
if the response status is configured in defaults.retry_statuses."""
retries = max(defaults.retries, 0)
for i in xrange(retries + 1):
try:
response, content = self._send_request(
uri, body=body, method=method, headers=headers
)
except Exception as err:
_logger.exception(
'Could not send request url=%s method=%s', uri, method)
raise HTTPError('HTTP Error %s' % str(err))
status = response.status
if self._is_need_retry(status, i, retries):
delay = 2 ** i
_logger.warning(
'The response status=%s of request which url=%s and'
' method=%s. Retry after %s seconds.',
status, uri, method, delay,
)
time.sleep(delay)
continue
return HTTPResponse(response, content)
package/bin/da_ess_contentupdate/cloudconnectlib/core/models.py
-277
View File
@@ -1,277 +0,0 @@
import base64
import traceback
from .ext import lookup_method
from .template import compile_template
from ..common.log import get_cc_logger
_logger = get_cc_logger()
class _Token(object):
"""Token class wraps a template expression"""
def __init__(self, source):
"""Constructs _Token from source. A rendered template
will be created if source is string type because Jinja
template must be a string."""
self._source = source
self._value_for = compile_template(source) \
if isinstance(source, basestring) else None
def render(self, variables):
"""Render value with variables if source is a string.
Otherwise return source directly."""
if self._value_for is None:
return self._source
try:
return self._value_for(variables)
except Exception as ex:
_logger.warning(
'Unable to render template "%s". Please make sure template is'
' a valid Jinja2 template and token is exist in variables. '
'message=%s cause=%s',
self._source,
ex.message,
traceback.format_exc()
)
return self._source
class DictToken(object):
"""DictToken wraps a dict which value is template expression"""
def __init__(self, template_expr):
self._tokens = {k: _Token(v)
for k, v in (template_expr or {}).iteritems()}
def render(self, variables):
return {k: v.render(variables) for k, v in self._tokens.iteritems()}
class BaseAuth(object):
"""A base class for all authorization classes"""
def __call__(self, headers, context):
raise NotImplementedError('Auth must be callable.')
class BasicAuthorization(BaseAuth):
"""BasicAuthorization class implements basic auth"""
def __init__(self, options):
if not options:
raise ValueError('Options for basic auth unexpected to be empty')
username = options.get('username')
if not username:
raise ValueError('Username is mandatory for basic auth')
password = options.get('password')
if not password:
raise ValueError('Password is mandatory for basic auth')
self._username = _Token(username)
self._password = _Token(password)
def __call__(self, headers, context):
username = self._username.render(context)
password = self._password.render(context)
headers['Authorization'] = 'Basic %s' % base64.encodestring(
username + ':' + password
).strip()
class Request(object):
def __init__(self, url, method, header=None, auth=None, body=None):
self._header = DictToken(header)
self._url = _Token(url)
self._method = method.upper()
self._auth = auth
self._body = DictToken(body)
@property
def header(self):
return self._header
@property
def url(self):
return self._url
@property
def method(self):
return self._method
@property
def auth(self):
return self._auth
@property
def body(self):
return self._body
def normalize_url(self, context):
"""Normalize url"""
return self._url.render(context)
def normalize_header(self, context):
"""Normalize headers which must be a dict which keys and values are
string."""
header = self.header.render(context)
return {k: str(v) for k, v in header.iteritems()}
def normalize_body(self, context):
"""Normalize body"""
return self.body.render(context)
class _Function(object):
def __init__(self, inputs, function):
self._inputs = tuple(_Token(expr) for expr in inputs or [])
self._function = function
@property
def inputs(self):
return self._inputs
def inputs_values(self, context):
"""
Get rendered input values.
"""
for arg in self._inputs:
yield arg.render(context)
@property
def function(self):
return self._function
class Task(_Function):
"""Task class wraps a task in processor pipeline"""
def __init__(self, inputs, function, output=None):
super(Task, self).__init__(inputs, function)
self._output = output
@property
def output(self):
return self._output
def execute(self, context):
"""Execute task with arguments which rendered from context """
args = [arg for arg in self.inputs_values(context)]
caller = lookup_method(self.function)
output = self._output
_logger.info(
'Executing task method: [%s], input size: [%s], output: [%s]',
self.function, len(args), output
)
if output is None:
caller(*args)
return {}
return {output: caller(*args)}
class Condition(_Function):
"""A condition return the value calculated from input and function"""
def calculate(self, context):
"""Calculate condition with input arguments rendered from context
and method which expected return a bool result.
:param context: context contains key value pairs
:return A bool value returned from the corresponding method
"""
args = [arg for arg in self.inputs_values(context)]
callable_method = lookup_method(self.function)
_logger.debug(
'Calculating condition with method: [%s], input size: [%s]',
self.function, len(args)
)
result = callable_method(*args)
_logger.debug("Calculated result: %s", result)
return result
class _Conditional(object):
"""A base class for all conditional action"""
def __init__(self, conditions):
self._conditions = conditions or []
@property
def conditions(self):
return self._conditions
def passed(self, context):
"""Determine if any condition is satisfied.
:param context: variables to render template
:return: `True` if all passed else `False`
"""
return any(
condition.calculate(context) for condition in self._conditions
)
class Processor(_Conditional):
"""Processor class contains a conditional data process pipeline"""
def __init__(self, skip_conditions, pipeline):
super(Processor, self).__init__(skip_conditions)
self._pipeline = pipeline or []
@property
def pipeline(self):
return self._pipeline
def should_skipped(self, context):
"""Determine processor if should skip process"""
return self.passed(context)
class IterationMode(_Conditional):
def __init__(self, iteration_count, conditions):
super(IterationMode, self).__init__(conditions)
self._iteration_count = iteration_count
@property
def iteration_count(self):
return self._iteration_count
@property
def conditions(self):
return self._conditions
class Checkpoint(object):
"""A checkpoint includes a namespace to determine the checkpoint location
and a content defined the format of content stored in checkpoint."""
def __init__(self, namespace, content):
"""Constructs checkpoint with given namespace and content template. """
if not content:
raise ValueError('Checkpoint content must not be empty')
self._namespace = tuple(_Token(expr) for expr in namespace or ())
self._content = DictToken(content)
@property
def namespace(self):
return self._namespace
def normalize_namespace(self, ctx):
"""Normalize namespace with context used to render template."""
return [token.render(ctx) for token in self._namespace]
@property
def content(self):
return self._content
def normalize_content(self, ctx):
"""Normalize checkpoint with context used to render template."""
return self._content.render(ctx)
package/bin/da_ess_contentupdate/cloudconnectlib/core/pipemgr.py
-14
View File
@@ -1,14 +0,0 @@
from solnlib.pattern import Singleton
class PipeManager(object):
__metaclass__ = Singleton
def __init__(self, event_writer=None):
self._event_writer = event_writer
def write_events(self, events):
if not self._event_writer:
print events
return True
return self._event_writer.write_events(events)
package/bin/da_ess_contentupdate/cloudconnectlib/core/template.py
-20
View File
@@ -1,20 +0,0 @@
from jinja2 import Template
import re
# This pattern matches the template with only one token inside like "{{
# token1}}", "{{ token2 }"
PATTERN = re.compile(r"^\{\{\s*(\w+)\s*\}\}$")
def compile_template(template):
_origin_template = template
_template = Template(template)
def translate_internal(context):
match = re.match(PATTERN, _origin_template)
if match:
context_var = context.get(match.groups()[0])
return context_var if context_var else ''
return _template.render(context)
return translate_internal
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/__init__.py
View File
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/cloud_connect_mod_input.py
-71
View File
@@ -1,71 +0,0 @@
import ConfigParser
import os.path as op
from .data_collection import ta_mod_input as ta_input
from .ta_cloud_connect_client import TACloudConnectClient as CollectorCls
from ..common.lib_util import (
get_main_file, get_app_root_dir, get_mod_input_script_name
)
def _load_options_from_inputs_spec(app_root, stanza_name):
input_spec_file = 'inputs.conf.spec'
file_path = op.join(app_root, 'README', input_spec_file)
if not op.isfile(file_path):
raise RuntimeError("README/%s doesn't exist" % input_spec_file)
parser = ConfigParser.RawConfigParser(allow_no_value=True)
parser.read(file_path)
options = parser.defaults().keys()
stanza_prefix = '%s://' % stanza_name
stanza_exist = False
for section in parser.sections():
if section == stanza_name or section.startswith(stanza_prefix):
options.extend(parser.options(section))
stanza_exist = True
if not stanza_exist:
raise RuntimeError("Stanza %s doesn't exist" % stanza_name)
return set(options)
def _find_ucc_global_config_json(app_root, ucc_config_filename):
"""Find UCC config file from all possible directories"""
candidates = ['local', 'default', 'bin',
op.join('appserver', 'static', 'js', 'build')]
for candidate in candidates:
file_path = op.join(app_root, candidate, ucc_config_filename)
if op.isfile(file_path):
return file_path
raise RuntimeError(
'Unable to load %s from [%s]'
% (ucc_config_filename, ','.join(candidates))
)
def _get_cloud_connect_config_json(script_name):
config_file_name = '.'.join([script_name, 'cc.json'])
return op.join(op.dirname(get_main_file()), config_file_name)
def run(single_instance=False):
script_name = get_mod_input_script_name()
cce_config_file = _get_cloud_connect_config_json(script_name)
app_root = get_app_root_dir()
ucc_config_path = _find_ucc_global_config_json(
app_root, 'globalConfig.json'
)
schema_params = _load_options_from_inputs_spec(app_root, script_name)
ta_input.main(
CollectorCls,
schema_file_path=ucc_config_path,
log_suffix=script_name,
cc_json_file=cce_config_file,
schema_para_list=schema_params,
single_instance=single_instance
)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/common/__init__.py
-52
View File
@@ -1,52 +0,0 @@
import json
import hashlib
def load_schema_file(schema_file):
"""
Load schema file.
"""
with open(schema_file) as f:
ret = json.load(f)
common = ret.get("_common_", dict())
if common:
for k, v in ret.items():
if k == "_common_" or not isinstance(v, dict):
continue
# merge common into other values
for _k, _v in common.items():
if _k not in v:
v[_k] = _v
ret[k] = v
return ret
def md5_of_dict(data):
"""
MD5 of dict data.
"""
md5 = hashlib.sha256()
if isinstance(data, dict):
for key in sorted(data.keys()):
md5.update(repr(key))
md5.update(md5_of_dict(data[key]))
elif isinstance(data, list):
for item in sorted(data):
md5.update(md5_of_dict(item))
else:
md5.update(repr(data))
return md5.hexdigest()
class UCCException(Exception):
"""
Dispatch engine exception.
"""
pass
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/common/log.py
-49
View File
@@ -1,49 +0,0 @@
import logging
from ...splunktalib.common import log as stclog
def set_log_level(log_level):
"""
Set log level.
"""
if isinstance(log_level, basestring):
if log_level.upper() == "DEBUG":
stclog.Logs().set_level(logging.DEBUG)
elif log_level.upper() == "INFO":
stclog.Logs().set_level(logging.INFO)
elif log_level.upper() == "WARN":
stclog.Logs().set_level(logging.WARN)
elif log_level.upper() == "ERROR":
stclog.Logs().set_level(logging.ERROR)
elif log_level.upper() == "WARNING":
stclog.Logs().set_level(logging.WARNING)
elif log_level.upper() == "CRITICAL":
stclog.Logs().set_level(logging.CRITICAL)
else:
stclog.Logs().set_level(logging.INFO)
elif isinstance(log_level, int):
if log_level in [logging.DEBUG, logging.INFO, logging.ERROR,
logging.WARN, logging.WARNING, logging.CRITICAL]:
stclog.Logs().set_level(log_level)
else:
stclog.Logs().set_level(logging.INFO)
else:
stclog.Logs().set_level(logging.INFO)
# Global logger
logger = stclog.Logs().get_logger("cloud_connect_engine")
def reset_logger(name):
"""
Reset logger.
"""
stclog.reset_logger(name)
global logger
logger = stclog.Logs().get_logger(name)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/common/rwlock.py
-71
View File
@@ -1,71 +0,0 @@
"""
This module provides Read-Write lock.
"""
import threading
class _ReadLocker(object):
def __init__(self, lock):
self.lock = lock
def __enter__(self):
self.lock.acquire_read()
def __exit__(self, exc_type, exc_val, exc_tb):
self.lock.release_read()
return False
class _WriteLocker(object):
def __init__(self, lock):
self.lock = lock
def __enter__(self):
self.lock.acquire_write()
def __exit__(self, exc_type, exc_val, exc_tb):
self.lock.release_write()
return False
class RWLock(object):
""" Simple Read-Write lock.
Allow multiple read but only one writing concurrently.
"""
def __init__(self):
self._condition = threading.Condition(threading.Lock())
self._readers = 0
def acquire_read(self):
self._condition.acquire()
self._readers += 1
self._condition.release()
def release_read(self):
self._condition.acquire()
try:
self._readers -= 1
if not self._readers:
self._condition.notifyAll()
finally:
self._condition.release()
def acquire_write(self):
self._condition.acquire()
while self._readers > 0:
self._condition.wait()
def release_write(self):
self._condition.release()
@property
def reader_lock(self):
return _ReadLocker(self)
@property
def writer_lock(self):
return _WriteLocker(self)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/common/schema_meta.py
-7
View File
@@ -1,7 +0,0 @@
FIELD_PRODUCT = '_product'
FIELD_REST_NAMESPACE = '_rest_namespace'
FIELD_REST_PREFIX = '_rest_prefix'
FIELD_PROTOCOL_VERSION = '_protocol_version'
FIELD_VERSION = '_version'
FIELD_ENCRYPTION_FORMATTER = '_encryption_formatter'
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/config.py
-367
View File
@@ -1,367 +0,0 @@
"""UCC Config Module
This is for load/save configuration in UCC server or TA.
The load/save action is based on specified schema.
"""
from __future__ import absolute_import
import json
import logging
import traceback
import time
from ..splunktalib.rest import splunkd_request, code_to_msg
from ..splunktalib.common import util as sc_util
from .common import log as stulog
from .common import UCCException
from urllib import quote
LOGGING_STOPPED = False
def stop_logging():
"""
Stop Config Logging. This is for not showing REST request error
while splunkd shutting down.
:return:
"""
global LOGGING_STOPPED
LOGGING_STOPPED = True
def log(msg, msgx='', level=logging.INFO, need_tb=False):
"""
Logging in UCC Config Module.
:param msg: message content
:param msgx: detail info.
:param level: logging level
:param need_tb: if need logging traceback
:return:
"""
global LOGGING_STOPPED
if LOGGING_STOPPED:
return
msgx = ' - ' + msgx if msgx else ''
content = 'UCC Config Module: %s%s' % (msg, msgx)
if need_tb:
stack = ''.join(traceback.format_stack())
content = '%s\r\n%s' % (content, stack)
stulog.logger.log(level, content, exc_info=1)
class ConfigException(UCCException):
"""Exception for UCC Config Exception
"""
pass
class Config(object):
"""UCC Config Module
"""
# Placeholder stands for any field
FIELD_PLACEHOLDER = '*'
# Head of non-processing endpoint
NON_PROC_ENDPOINT = '#'
# Some meta fields in UCC Config schema
META_FIELDS = ('_product', '_rest_namespace', '_rest_prefix',
'_protocol_version', '_version',
'_encryption_formatter')
# Default Values for Meta fields
META_FIELDS_DEFAULT = {
'_encryption_formatter': '',
}
def __init__(self, splunkd_uri, session_key, schema,
user='nobody', app='-'):
"""
:param splunkd_uri: the root uri of Splunk server,
like https://127.0.0.1:8089
:param session_key: session key for Splunk server
:param schema:
:param user: owner of the resources requested
:param app: namespace of the resources requested
:return:
"""
self.splunkd_uri = splunkd_uri.strip('/')
self.session_key = session_key
self.user, self.app = user, app
self._parse_schema(schema)
self._check_protocol_version()
def load(self):
"""Load Configurations in UCC according to the schema
It will raise exception if failing to load any endpoint,
because it make no sense with not complete configuration info.
"""
log('"load" method in', level=logging.DEBUG)
ret = {meta_field: getattr(self, meta_field)
for meta_field in Config.META_FIELDS}
for ep_id, ep in self._endpoints.iteritems():
data = {'output_mode': 'json', '--cred--': '1'}
retries = 4
waiting_time = [1, 2, 2]
for retry in xrange(retries):
resp, cont = splunkd_request(
splunkd_uri=self.make_uri(ep_id),
session_key=self.session_key,
data=data,
retry=3
)
if resp is None or resp.status != 200:
msg = 'Fail to load endpoint "{ep_id}" - {err}' \
''.format(ep_id=ep_id,
err=code_to_msg(resp, cont)
if resp else cont)
log(msg, level=logging.ERROR, need_tb=True)
raise ConfigException(msg)
try:
ret[ep_id] = self._parse_content(ep_id, cont)
except ConfigException, exc:
log(exc, level=logging.WARNING, need_tb=True)
if retry < retries-1:
time.sleep(waiting_time[retry])
else:
break
else:
log(exc, level=logging.ERROR, need_tb=True)
raise exc
log('"load" method out', level=logging.DEBUG)
return ret
def update_items(self, endpoint_id, item_names, field_names, data,
raise_if_failed=False):
"""Update items in specified endpoint with given fields in data
:param endpoint_id: endpoint id in schema, the key name in schema
:param item_names: a list of item name
:param field_names: a list of updated fields
:param data: a dict of content for items, for example:
{
"item_name_1": {
"field_name_1": "value_1",
"field_name_2": "value_2",
},
"item_name_2": {
"field_name_1": "value_1x",
"field_name_2": "value_2x",
}
}
:raise_if_failed: raise an exception if updating failed.
:return: a list of endpoint ids, which are failed to be updated.
If raise_if_failed is True, it will exist with an exception
on any updating failed.
"""
log('"update_items" method in',
msgx='endpoint_id=%s, item_names=%s, field_names=%s'
% (endpoint_id, item_names, field_names),
level=logging.DEBUG)
assert endpoint_id in self._endpoints, \
'Unexpected endpoint id in given schema - {ep_id}' \
''.format(ep_id=endpoint_id)
item_names_failed = []
for item_name in item_names:
item_data = data.get(item_name, {})
item_data = {field_name: self.dump_value(endpoint_id,
item_name,
field_name,
item_data[field_name])
for field_name in field_names
if field_name in item_data}
if not item_data:
continue
item_uri = self.make_uri(endpoint_id, item_name=item_name)
resp, cont = splunkd_request(splunkd_uri=item_uri,
session_key=self.session_key,
data=item_data,
method="POST",
retry=3
)
if resp is None or resp.status not in (200, 201):
msg = 'Fail to update item "{item}" in endpoint "{ep_id}"' \
' - {err}'.format(ep_id=endpoint_id,
item=item_name,
err=code_to_msg(resp, cont)
if resp else cont)
log(msg, level=logging.ERROR)
if raise_if_failed:
raise ConfigException(msg)
item_names_failed.append(item_name)
log('"update_items" method out', level=logging.DEBUG)
return item_names_failed
def make_uri(self, endpoint_id, item_name=None):
"""Make uri for REST endpoint in TA according to given schema
:param endpoint_id: endpoint id in schema
:param item_name: item name for given endpoint. None for listing all
:return:
"""
endpoint = self._endpoints[endpoint_id]['endpoint']
ep_full = endpoint[1:].strip('/') \
if endpoint.startswith(Config.NON_PROC_ENDPOINT) else \
'{admin_match}/{protocol_version}/{endpoint}' \
''.format(admin_match=self._rest_namespace,
protocol_version=self._protocol_version,
endpoint=(self._rest_prefix +
self._endpoints[endpoint_id]['endpoint']))
ep_uri = None if endpoint_id not in self._endpoints else \
'{splunkd_uri}/servicesNS/{user}/{app}/{endpoint_full}' \
''.format(splunkd_uri=self.splunkd_uri,
user=self.user,
app=self.app,
endpoint_full=ep_full
)
url = ep_uri if item_name is None else "{ep_uri}/{item_name}"\
.format(ep_uri=ep_uri, item_name=quote(item_name))
if item_name is None:
url += '?count=-1'
log('"make_uri" method', msgx='url=%s' % url,
level=logging.DEBUG)
return url
def _parse_content(self, endpoint_id, content):
"""Parse content returned from REST
:param content: a JSON string returned from REST.
"""
try:
content = json.loads(content)['entry']
ret = {ent['name']: ent['content'] for ent in content}
except Exception as exc:
msg = 'Fail to parse content from endpoint_id=%s' \
' - %s' % (endpoint_id, exc)
log(msg, level=logging.ERROR, need_tb=True)
raise ConfigException(msg)
ret = {name: {key: self.load_value(endpoint_id, name, key, val)
for key, val in ent.iteritems()
if not key.startswith('eai:')}
for name, ent in ret.iteritems()}
return ret
def _parse_schema(self, ucc_config_schema):
try:
ucc_config_schema = json.loads(ucc_config_schema)
except ValueError:
msg = 'Invalid JSON content of schema'
log(msg, level=logging.ERROR, need_tb=True)
raise ConfigException(msg)
except Exception as exc:
log(exc, level=logging.ERROR, need_tb=True)
raise ConfigException(exc)
ucc_config_schema.update({key: val for key, val in
Config.META_FIELDS_DEFAULT.iteritems()
if key not in ucc_config_schema})
for field in Config.META_FIELDS:
assert field in ucc_config_schema and \
isinstance(ucc_config_schema[field], basestring), \
'Missing or invalid field "%s" in given schema' % field
setattr(self, field, ucc_config_schema[field])
self._endpoints = {}
for key, val in ucc_config_schema.iteritems():
if key.startswith('_'):
continue
assert isinstance(val, dict), \
'The schema of endpoint "%s" should be dict' % key
assert 'endpoint' in val, \
'The endpoint "%s" has no endpoint entry' % key
self._endpoints[key] = val
def _check_protocol_version(self):
"""
Check if the protocol version in given schema is supported.
:return:
"""
if not self._protocol_version:
return
if not self._protocol_version.startswith('1.'):
raise ConfigException('Unsupported protocol version "%s" '
'in given schema' % self._protocol_version)
def load_value(self, endpoint_id, item_name, fname, fval):
field_type = self._get_field_type(endpoint_id, item_name, fname)
if field_type == '':
return fval
try:
field_type = field_type.lower()
if field_type == 'bool':
return True if sc_util.is_true(fval) else False
elif field_type == 'int':
return int(fval)
elif field_type == 'json':
return json.loads(fval)
except Exception as exc:
msg = 'Fail to load value of "{type_name}" - ' \
'endpoint={endpoint}, item={item}, field={field}' \
''.format(type_name=field_type,
endpoint=endpoint_id,
item=item_name,
field=fname)
log(msg, msgx=str(exc), level=logging.WARNING, need_tb=True)
raise ConfigException(msg)
def dump_value(self, endpoint_id, item_name, fname, fval):
field_type = self._get_field_type(endpoint_id, item_name, fname)
if field_type == '':
return fval
try:
field_type = field_type.lower()
if field_type == 'bool':
return str(fval).lower()
elif field_type == 'json':
return json.dumps(fval)
else:
return fval
except Exception, exc:
msg = 'Fail to dump value of "{type_name}" - ' \
'endpoint={endpoint}, item={item}, field={field}' \
''.format(type_name=field_type,
endpoint=endpoint_id,
item=item_name,
field=fname)
log(msg, msgx=str(exc), level=logging.ERROR, need_tb=True)
raise ConfigException(msg)
def _get_field_type(self, endpoint_id, item_name, fname):
field_types = self._endpoints[endpoint_id].get('field_types', {})
if item_name in field_types:
fields = field_types[item_name]
elif Config.FIELD_PLACEHOLDER in field_types:
fields = field_types[Config.FIELD_PLACEHOLDER]
else:
fields = {}
field_type = fields.get(fname, '')
if field_type not in ('', 'bool', 'int', 'json'):
msg = 'Unsupported type "{type_name}" for value in schema - ' \
'endpoint={endpoint}, item={item}, field={field}' \
''.format(type_name=field_type,
endpoint=endpoint_id,
item=item_name,
field=fname)
log(msg, level=logging.ERROR, need_tb=True)
raise ConfigException(msg)
return field_type
def get_endpoints(self):
return self._endpoints
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/__init__.py
-1
View File
@@ -1 +0,0 @@
__version__ = "1.0.2"
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_checkpoint_manager.py
-147
View File
@@ -1,147 +0,0 @@
import json
import re
from . import ta_consts as c
from . import ta_helper as th
from ..common import log as stulog
from ...splunktalib import state_store as ss
from ...splunktalib.common.util import is_true
class TACheckPointMgr(object):
SEPARATOR = "_" * 3
# FIXME We'd better move all default values together
_DEFAULT_MAX_CACHE_SECONDS = 5
_MAXIMUM_MAX_CACHE_SECONDS = 3600
def __init__(self, meta_config, task_config):
self._task_config = task_config
self._store = self._create_state_store(
meta_config,
task_config.get(c.checkpoint_storage_type),
task_config[c.appname]
)
def _create_state_store(self, meta_config, storage_type, app_name):
stulog.logger.debug('Got checkpoint storage type=%s', storage_type)
if storage_type == c.checkpoint_kv_storage:
collection_name = self._get_collection_name()
stulog.logger.debug(
'Creating KV state store, collection name=%s', collection_name
)
return ss.get_state_store(
meta_config,
appname=app_name,
collection_name=collection_name,
use_kv_store=True
)
use_cache_file = self._use_cache_file()
max_cache_seconds = \
self._get_max_cache_seconds() if use_cache_file else None
stulog.logger.debug(
'Creating file state store, use_cache_file=%s, max_cache_seconds=%s',
use_cache_file, max_cache_seconds
)
return ss.get_state_store(
meta_config,
app_name,
use_cache_file=use_cache_file,
max_cache_seconds=max_cache_seconds
)
def _get_collection_name(self):
collection = self._task_config.get(c.collection_name)
collection = collection.strip() if collection else ''
if not collection:
input_name = self._task_config[c.mod_input_name]
stulog.logger.info(
'Collection name="%s" is empty, set it to "%s"',
collection, input_name
)
collection = input_name
return re.sub(r'[^\w]+', '_', collection)
def _use_cache_file(self):
# TODO Move the default value outside code
use_cache_file = is_true(self._task_config.get(c.use_cache_file, True))
if use_cache_file:
stulog.logger.info(
"Stanza=%s using cached file store to create checkpoint",
self._task_config[c.stanza_name]
)
return use_cache_file
def _get_max_cache_seconds(self):
default = self._DEFAULT_MAX_CACHE_SECONDS
seconds = self._task_config.get(
c.max_cache_seconds, default
)
try:
seconds = int(seconds)
except ValueError:
stulog.logger.warning(
"The max_cache_seconds '%s' is not a valid integer,"
" so set this variable to default value %s",
seconds, default
)
seconds = default
else:
maximum = self._MAXIMUM_MAX_CACHE_SECONDS
if not (1 <= seconds <= maximum):
# for seconds>3600 set it to 3600. for seconds <=0 set it to default.
adjusted = max(min(seconds, maximum), default)
stulog.logger.warning(
"The max_cache_seconds (%s) is expected in range[1,%s],"
" set it to %s",
seconds, maximum, adjusted
)
seconds = adjusted
return seconds
def get_ckpt_key(self, namespaces=None):
return self._key_formatter(namespaces)
def get_ckpt(self, namespaces=None, show_namespaces=False):
key, namespaces = self.get_ckpt_key(namespaces)
raw_checkpoint = self._store.get_state(key)
stulog.logger.info("Get checkpoint key='%s' value='%s'",
key, json.dumps(raw_checkpoint))
if not show_namespaces and raw_checkpoint:
return raw_checkpoint.get("data")
return raw_checkpoint
def update_ckpt(self, ckpt, namespaces=None):
if not ckpt:
stulog.logger.warning("Checkpoint expect to be not empty.")
return
key, namespaces = self.get_ckpt_key(namespaces)
value = {"namespaces": namespaces, "data": ckpt}
stulog.logger.info("Update checkpoint key='%s' value='%s'",
key, json.dumps(value))
self._store.update_state(key, value)
def remove_ckpt(self, namespaces=None):
key, namespaces = self.get_ckpt_key(namespaces)
self._store.delete_state(key)
def _key_formatter(self, namespaces=None):
if not namespaces:
stulog.logger.info('Namespaces is empty, using stanza name instead.')
namespaces = [self._task_config[c.stanza_name]]
key_str = TACheckPointMgr.SEPARATOR.join(namespaces)
hashed_file = th.format_name_for_file(key_str)
stulog.logger.info("raw_file='%s' hashed_file='%s'", key_str, hashed_file)
return hashed_file, namespaces
def close(self, key=None):
try:
self._store.close(key)
stulog.logger.info('Closed state store successfully. key=%s', key)
except Exception:
stulog.logger.exception('Error closing state store. key=%s', key)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_config.py
-155
View File
@@ -1,155 +0,0 @@
import os.path as op
import socket
import ta_consts as c
import ta_helper as th
from ..common import log as stulog
from ...splunktalib import modinput as modinput
from ...splunktalib import splunk_cluster as sc
from ...splunktalib.common import util
# methods can be overrided by subclass : process_task_configs
class TaConfig(object):
_current_hostname = socket.gethostname()
_appname = util.get_appname_from_path(op.abspath(__file__))
def __init__(self, meta_config, client_schema, log_suffix=None,
stanza_name=None, input_type=None,
single_instance=True):
self._meta_config = meta_config
self._stanza_name = stanza_name
self._input_type = input_type
self._log_suffix = log_suffix
self._single_instance = single_instance
self._task_configs = []
self._client_schema = client_schema
self._server_info = sc.ServerInfo(meta_config[c.server_uri],
meta_config[c.session_key])
self._all_conf_contents = {}
self._get_division_settings = {}
self.set_logging()
self._load_task_configs()
def is_shc_member(self):
return self._server_info.is_shc_member()
def is_search_head(self):
return self._server_info.is_search_head()
def is_single_instance(self):
return self._single_instance
def get_meta_config(self):
return self._meta_config
def get_task_configs(self):
return self._task_configs
def get_all_conf_contents(self):
if self._all_conf_contents:
return self._all_conf_contents.get(c.inputs), \
self._all_conf_contents.get(c.all_configs), \
self._all_conf_contents.get(c.global_settings)
inputs, configs, global_settings = th.get_all_conf_contents(
self._meta_config[c.server_uri],
self._meta_config[c.session_key],
self._client_schema, self._input_type)
self._all_conf_contents[c.inputs] = inputs
self._all_conf_contents[c.all_configs] = configs
self._all_conf_contents[c.global_settings] = global_settings
return inputs, configs, global_settings
def set_logging(self):
# The default logger name is "cloud_connect_engine"
if self._stanza_name and self._log_suffix:
logger_name = self._log_suffix + "_" + th.format_name_for_file(
self._stanza_name)
stulog.reset_logger(logger_name)
inputs, configs, global_settings = self.get_all_conf_contents()
log_level = "INFO"
for item in global_settings.get("settings"):
if item.get(c.name) == "logging" and item.get("loglevel"):
log_level = item["loglevel"]
break
stulog.set_log_level(log_level)
stulog.logger.info("Set log_level={}".format(log_level))
stulog.logger.info("Start {} task".format(self._stanza_name))
def get_input_type(self):
return self._input_type
def _get_checkpoint_storage_type(self, config):
cs_type = config.get(c.checkpoint_storage_type)
stulog.logger.debug("Checkpoint storage type=%s", cs_type)
cs_type = cs_type.strip() if cs_type else c.checkpoint_auto
# Allow user configure 'auto' and 'file' only.
if cs_type not in (c.checkpoint_auto, c.checkpoint_file):
stulog.logger.warning(
"Checkpoint storage type='%s' is invalid, change it to '%s'",
cs_type, c.checkpoint_auto
)
cs_type = c.checkpoint_auto
if cs_type == c.checkpoint_auto and self.is_search_head():
stulog.logger.info(
"Checkpoint storage type is '%s' and instance is "
"search head, set checkpoint storage type to '%s'.",
c.checkpoint_auto,
c.checkpoint_kv_storage
)
cs_type = c.checkpoint_kv_storage
return cs_type
def _load_task_configs(self):
inputs, configs, global_settings = self.get_all_conf_contents()
if self._input_type:
inputs = inputs.get(self._input_type)
if not self._single_instance:
inputs = [input for input in inputs if
input[c.name] == self._stanza_name]
all_task_configs = []
for input in inputs:
task_config = {}
task_config.update(input)
task_config[c.configs] = configs
task_config[c.settings] = \
{item[c.name]: item for item in global_settings["settings"]}
if self.is_single_instance():
collection_interval = "collection_interval"
task_config[c.interval] = task_config.get(collection_interval)
task_config[c.interval] = int(task_config[c.interval])
if task_config[c.interval] <= 0:
raise ValueError(
"The interval value {} is invalid."
" It should be a positive integer".format(
task_config[c.interval]))
task_config[c.checkpoint_storage_type] = \
self._get_checkpoint_storage_type(task_config)
task_config[c.appname] = TaConfig._appname
task_config[c.mod_input_name] = self._input_type
task_config[c.stanza_name] = task_config[c.name]
all_task_configs.append(task_config)
self._task_configs = all_task_configs
# Override this method if some transforms or validations needs to be done
# before task_configs is exposed
def process_task_configs(self, task_configs):
pass
def create_ta_config(settings, config_cls=TaConfig, log_suffix=None,
single_instance=True):
meta_config, configs = modinput.get_modinput_configs_from_stdin()
stanza_name = None
input_type = None
if configs and "://" in configs[0].get("name", ""):
input_type, stanza_name = configs[0].get("name").split("://", 1)
return config_cls(meta_config, settings, log_suffix, stanza_name,
input_type, single_instance=single_instance)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_consts.py
-54
View File
@@ -1,54 +0,0 @@
server_uri = "server_uri"
session_key = "session_key"
version = "version"
appname = "appname"
event_writer = "event_writer"
index = "index"
default_index = "default"
source = "source"
sourcetype = "sourcetype"
data_loader = "data_loader"
meta_configs = "meta_configs"
disabled = "disabled"
resource = "resource"
events = "events"
scope = "scope"
checkpoint_dir = "checkpoint_dir"
ckpt_dict = "ckpt_dict"
inputs = "inputs"
input_name = "input_name"
input_data = "input_data"
interval = "interval"
data = "data"
batch_size = 'batch_size'
time_fmt = "%Y-%m-%dT%H:%M:%S"
utc_time_fmt = "%Y-%m-%dT%H:%M:%S.%fZ"
# system setting keys
checkpoint_storage_type = "builtin_system_checkpoint_storage_type"
# Possible values for checkpoint storage type
checkpoint_auto = 'auto'
checkpoint_kv_storage = 'kv_store'
checkpoint_file = 'file'
# For cache file
use_cache_file = "builtin_system_use_cache_file"
max_cache_seconds = "builtin_system_max_cache_seconds"
# For kv store
collection_name = "builtin_system_kvstore_collection_name"
settings = "__settings__"
configs = "__configs__"
name = "name"
config = "config"
division = "division"
stanza_name = "stanza_name"
divide_key = "_divide_key"
divide_endpoint = "_divide_endpoint"
mod_input_name = "mod_input_name"
global_settings = "global_settings"
all_configs = "all_configs"
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_data_client.py
-84
View File
@@ -1,84 +0,0 @@
#!/usr/bin/python
from . import ta_checkpoint_manager as cp
from . import ta_data_collector as tdc
def build_event(host=None,
source=None,
sourcetype=None,
time=None,
index=None,
raw_data="",
is_unbroken=False,
is_done=False):
if is_unbroken is False and is_done is True:
raise Exception('is_unbroken=False is_done=True is invalid')
return tdc.event_tuple._make([host, source, sourcetype, time, index,
raw_data, is_unbroken, is_done])
class TaDataClient(object):
def __init__(self,
meta_config,
task_config,
checkpoint_mgr=None,
event_writer=None):
self._meta_config = meta_config
self._task_config = task_config
self._checkpoint_mgr = checkpoint_mgr
self._event_writer = event_writer
self._stop = False
def is_stopped(self):
return self._stop
def stop(self):
self._stop = True
def get(self):
raise StopIteration
def create_data_collector(dataloader,
tconfig,
meta_configs,
task_config,
data_client_cls,
checkpoint_cls=None):
checkpoint_manager_cls = checkpoint_cls or cp.TACheckPointMgr
return tdc.TADataCollector(tconfig, meta_configs, task_config,
checkpoint_manager_cls, data_client_cls,
dataloader)
def client_adatper(job_func):
class TaDataClientAdapter(TaDataClient):
def __init__(self, all_conf_contents, meta_config, task_config,
chp_mgr):
super(TaDataClientAdapter, self).__init__(meta_config, task_config,
chp_mgr)
self._execute_times = 0
self._gen = job_func(self._task_config, chp_mgr)
def stop(self):
"""
overwrite to handle stop control command
"""
# normaly base class just set self._stop as True
super(TaDataClientAdapter, self).stop()
def get(self):
"""
overwrite to get events
"""
self._execute_times += 1
if self.is_stopped():
# send stop signal
self._gen.send(self.is_stopped())
raise StopIteration
if self._execute_times == 1:
return self._gen.next()
return self._gen.send(self.is_stopped())
return TaDataClientAdapter
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_data_collector.py
-144
View File
@@ -1,144 +0,0 @@
#!/usr/bin/python
import threading
import time
from collections import namedtuple
import ta_consts as c
from ..common import log as stulog
from ...splunktalib.common import util as scu
evt_fmt = ("<stream><event><host>{0}</host>"
"<source><![CDATA[{1}]]></source>"
"<sourcetype><![CDATA[{2}]]></sourcetype>"
"<time>{3}</time>"
"<index>{4}</index><data>"
"<![CDATA[{5}]]></data></event></stream>")
unbroken_evt_fmt = ("<stream>"
"<event unbroken=\"1\">"
"<host>{0}</host>"
"<source><![CDATA[{1}]]></source>"
"<sourcetype><![CDATA[{2}]]></sourcetype>"
"<time>{3}</time>"
"<index>{4}</index>"
"<data><![CDATA[{5}]]></data>"
"{6}"
"</event>"
"</stream>")
event_tuple = namedtuple('Event',
['host', 'source', 'sourcetype', 'time', 'index',
'raw_data', 'is_unbroken', 'is_done'])
class TADataCollector(object):
def __init__(self, tconfig, meta_config, task_config,
checkpoint_manager_cls, data_client_cls, data_loader):
self._lock = threading.Lock()
self._ta_config = tconfig
self._meta_config = meta_config
self._task_config = task_config
self._stopped = False
self._p = self._get_logger_prefix()
self._checkpoint_manager = checkpoint_manager_cls(meta_config,
task_config)
self.data_client_cls = data_client_cls
self._data_loader = data_loader
self._client = None
def get_meta_configs(self):
return self._meta_config
def get_task_config(self):
return self._task_config
def get_interval(self):
return self._task_config[c.interval]
def _get_logger_prefix(self):
pairs = ['{}="{}"'.format(c.stanza_name, self._task_config[
c.stanza_name])]
return "[{}]".format(" ".join(pairs))
def stop(self):
self._stopped = True
if self._client:
self._client.stop()
def __call__(self):
self.index_data()
def _build_event(self, events):
if not events:
return None
if not isinstance(events, list):
events = [events]
evts = []
for event in events:
assert event.raw_data, "the raw data of events is empty"
if event.is_unbroken:
evt = unbroken_evt_fmt.format(
event.host or "", event.source or "", event.sourcetype or
"", event.time or "", event.index or "",
scu.escape_cdata(event.raw_data), "<done/>" if
event.is_done else "")
else:
evt = evt_fmt.format(event.host or "", event.source or "",
event.sourcetype or "", event.time or "",
event.index or "",
scu.escape_cdata(event.raw_data))
evts.append(evt)
return evts
def _create_data_client(self):
return self.data_client_cls(self._meta_config,
self._task_config,
self._checkpoint_manager,
self._data_loader.get_event_writer())
def index_data(self):
if self._lock.locked():
stulog.logger.debug(
"Last round of stanza={} is not done yet".format(
self._task_config[c.stanza_name]))
return
with self._lock:
try:
self._do_safe_index()
self._checkpoint_manager.close()
except Exception:
stulog.logger.exception("{} Failed to index data"
.format(self._p))
stulog.logger.info("{} End of indexing data".format(self._p))
if not self._ta_config.is_single_instance():
self._data_loader.tear_down()
def _write_events(self, events):
evts = self._build_event(events)
if evts:
if not self._data_loader.write_events(evts):
stulog.logger.info("{} the event queue is closed and the "
"received data will be discarded".format(
self._p))
return False
return True
def _do_safe_index(self):
self._client = self._create_data_client()
while not self._stopped:
try:
events = self._client.get()
if not events:
continue
else:
if not self._write_events(events):
break
except StopIteration:
stulog.logger.info("{} Finished this round".format(self._p))
return
except Exception:
stulog.logger.exception("{} Failed to get msg".format(self._p))
break
# in case encounter exception or fail to write events
if not self._stopped:
self.stop()
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_data_loader.py
-168
View File
@@ -1,168 +0,0 @@
"""
Data Loader main entry point
"""
import Queue
import os.path as op
import ConfigParser
from ...splunktalib.concurrent import concurrent_executor as ce
from ...splunktalib import timer_queue as tq
from ...splunktalib.schedule import job as sjob
from ...splunktalib.common import log
class TADataLoader(object):
"""
Data Loader boots all underlying facilities to handle data collection
"""
def __init__(self, job_scheduler, event_writer):
"""
@configs: a list like object containing a list of dict
like object. Each element shall implement dict.get/[] like interfaces
to get the value for a key.
@job_scheduler: schedulering the jobs. shall implement get_ready_jobs
@event_writer: write_events
"""
self._settings = self._read_default_settings()
self._settings["daemonize_thread"] = False
self._event_writer = event_writer
self._wakeup_queue = Queue.Queue()
self._scheduler = job_scheduler
self._timer_queue = tq.TimerQueue()
self._executor = ce.ConcurrentExecutor(self._settings)
self._started = False
def run(self, jobs):
if self._started:
return
self._started = True
self._event_writer.start()
self._executor.start()
self._timer_queue.start()
self._scheduler.start()
log.logger.info("TADataLoader started.")
def _enqueue_io_job(job):
job_props = job.get_props()
real_job = job_props["real_job"]
self.run_io_jobs((real_job,))
for job in jobs:
j = sjob.Job(_enqueue_io_job, {"real_job": job},
job.get_interval())
self._scheduler.add_jobs((j,))
self._wait_for_tear_down()
for job in jobs:
job.stop()
self._scheduler.tear_down()
self._timer_queue.tear_down()
self._executor.tear_down()
self._event_writer.tear_down()
log.logger.info("DataLoader stopped.")
def _wait_for_tear_down(self):
wakeup_q = self._wakeup_queue
while 1:
try:
go_exit = wakeup_q.get(timeout=1)
except Queue.Empty:
pass
else:
if go_exit:
log.logger.info("DataLoader got stop signal")
self._stopped = True
break
def tear_down(self):
self._wakeup_queue.put(True)
log.logger.info("DataLoader is going to stop.")
def stopped(self):
return self._stopped
def run_io_jobs(self, jobs, block=True):
self._executor.enqueue_io_funcs(jobs, block)
def run_compute_job(self, func, args=(), kwargs={}):
self._executor.run_compute_func_sync(func, args, kwargs)
def run_compute_job_async(self, func, args=(), kwargs={}, callback=None):
"""
@return: AsyncResult
"""
return self._executor.run_compute_func_async(func, args,
kwargs, callback)
def add_timer(self, callback, when, interval):
return self._timer_queue.add_timer(callback, when, interval)
def remove_timer(self, timer):
self._timer_queue.remove_timer(timer)
def write_events(self, events):
return self._event_writer.write_events(events)
def get_event_writer(self):
return self._event_writer
@staticmethod
def _read_default_settings():
cur_dir = op.dirname(op.abspath(__file__))
setting_file = op.join(cur_dir,"../../","splunktalib", "setting.conf")
parser = ConfigParser.ConfigParser()
parser.read(setting_file)
settings = {}
keys = ("process_size", "thread_min_size", "thread_max_size",
"task_queue_size")
for option in keys:
try:
settings[option] = parser.get("global", option)
except ConfigParser.NoOptionError:
settings[option] = -1
try:
settings[option] = int(settings[option])
except ValueError:
settings[option] = -1
log.logger.debug("settings: %s", settings)
return settings
class GlobalDataLoader(object):
""" Singleton, inited when started"""
__instance = None
@staticmethod
def get_data_loader(scheduler, writer):
if GlobalDataLoader.__instance is None:
GlobalDataLoader.__instance = TADataLoader(
scheduler, writer)
return GlobalDataLoader.__instance
@staticmethod
def reset():
GlobalDataLoader.__instance = None
def create_data_loader():
"""
create a data loader with default event_writer, job_scheudler
"""
from ...splunktalib import event_writer as ew
from ...splunktalib.schedule import scheduler as sched
writer = ew.EventWriter()
scheduler = sched.Scheduler()
loader = GlobalDataLoader.get_data_loader(scheduler, writer)
return loader
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_helper.py
-154
View File
@@ -1,154 +0,0 @@
import hashlib
import json
import os.path as op
import re
from calendar import timegm
from datetime import datetime
import functools32
from splunktaucclib.global_config import GlobalConfig, GlobalConfigSchema
from . import ta_consts as c
from ...splunktacollectorlib import config as sc
from ...splunktalib.common import util
def utc2timestamp(human_time):
regex1 = ur"\d{4}-\d{2}-\d{2}.\d{2}:\d{2}:\d{2}"
match = re.search(regex1, human_time)
if match:
formated = match.group()
else:
return None
strped_time = datetime.strptime(formated, c.time_fmt)
timestamp = timegm(strped_time.utctimetuple())
regex2 = "\d{4}-\d{2}-\d{2}.\d{2}:\d{2}:\d{2}(\.\d+)"
match = re.search(regex2, human_time)
if match:
timestamp += float(match.group(1))
else:
timestamp += float("0.000000")
return timestamp
def get_md5(data):
"""
function name is not change, actually use sha1 instead
:param data:
:return:
"""
assert data is not None, "The input cannot be None"
if isinstance(data, (unicode, str)):
return hashlib.sha256(data.encode('utf-8')).hexdigest()
elif isinstance(data, (list, tuple, dict)):
return hashlib.sha256(json.dumps(data).encode('utf-8')).hexdigest()
def get_all_conf_contents(server_uri, sessionkey, settings, input_type=None):
schema = GlobalConfigSchema(settings)
global_config = GlobalConfig(
server_uri, sessionkey, schema
)
inputs = global_config.inputs.load(input_type=input_type)
configs = global_config.configs.load()
settings = global_config.settings.load()
return inputs, configs, settings
@functools32.lru_cache(maxsize=64)
def format_name_for_file(name):
return hashlib.sha256(name).hexdigest()
class ConfigSchemaHandler(object):
_app_name = util.get_appname_from_path(op.abspath(__file__))
# Division schema keys.
TYPE = "type"
TYPE_SINGLE = "single"
TYPE_MULTI = "multi"
REFER = "refer"
SEPARATOR = "separator"
def __init__(self, meta_configs, client_schema):
self._config = sc.Config(splunkd_uri=meta_configs[c.server_uri],
session_key=meta_configs[c.session_key],
schema=json.dumps(client_schema[
c.config]),
user="nobody",
app=ConfigSchemaHandler._app_name)
self._client_schema = client_schema
self._all_conf_contents = {}
self._load_conf_contents()
self._division_settings = self._divide_settings()
def get_endpoints(self):
return self._config.get_endpoints()
def get_all_conf_contents(self):
return self._all_conf_contents
def get_single_conf_contents(self, endpoint):
return self._all_conf_contents.get(endpoint)
def get_division_settings(self):
return self._division_settings
def _divide_settings(self):
division_schema = self._client_schema[c.division]
division_settings = dict()
for division_endpoint, division_contents in division_schema.iteritems():
division_settings[division_endpoint] = self._process_division(
division_endpoint, division_contents)
return division_settings
def _load_conf_contents(self):
self._all_conf_contents = self._config.load()
def _process_division(self, division_endpoint, division_contents):
division_metrics = []
assert isinstance(division_contents, dict)
for division_key, division_value in division_contents.iteritems():
try:
assert self.TYPE in division_value and \
division_value[self.TYPE] in \
[self.TYPE_SINGLE, self.TYPE_MULTI] and \
self.SEPARATOR in division_value if \
division_value[self.TYPE] == self.TYPE_MULTI else True
except Exception:
raise Exception("Invalid division schema")
division_metrics.append(DivisionRule(division_endpoint,
division_key,
division_value[self.TYPE],
division_value.get(
self.SEPARATOR,
),
division_value.get(
self.REFER,
)))
return division_metrics
class DivisionRule(object):
def __init__(self, endpoint, metric, type, separator, refer):
self._endpoint = endpoint
self._metric = metric
self._type = type
self._separator = separator
self._refer = refer
def endpoint(self):
return self._endpoint
def metric(self):
return self._metric
def type(self):
return self._type
def separator(self):
return self._separator
def refer(self):
return self._refer
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/data_collection/ta_mod_input.py
-272
View File
@@ -1,272 +0,0 @@
#!/usr/bin/python
"""
This is the main entry point for My TA
"""
import os.path as op
import platform
import sys
import time
from . import ta_checkpoint_manager as cpmgr
from . import ta_config as tc
from . import ta_data_client as tdc
from . import ta_data_loader as dl
from ..common import load_schema_file as ld
from ..common import log as stulog
from ...common.lib_util import get_app_root_dir, get_mod_input_script_name
from ...splunktalib import file_monitor as fm
from ...splunktalib import modinput
from ...splunktalib import orphan_process_monitor as opm
from ...splunktalib.common import util as utils
utils.remove_http_proxy_env_vars()
__CHECKPOINT_DIR_MAX_LEN__ = 180
def do_scheme(
mod_input_name,
schema_para_list=None,
single_instance=True,
):
"""
Feed splunkd the TA's scheme
"""
builtin_names = {
"name", "index", "sourcetype", "host", "source",
"disabled", "interval"
}
param_string_list = []
if schema_para_list is None:
schema_para_list = ()
for param in schema_para_list:
if param in builtin_names:
continue
param_string_list.append(
"""
<arg name="{param}">
<title>{param}</title>
<required_on_create>0</required_on_create>
<required_on_edit>0</required_on_edit>
</arg>
""".format(param=param)
)
description = ("Go to the add-on's configuration UI and configure"
" modular inputs under the Inputs menu.")
print """
<scheme>
<title>{data_input_title}</title>
<description>{description}</description>
<use_external_validation>true</use_external_validation>
<streaming_mode>xml</streaming_mode>
<use_single_instance>{single_instance}</use_single_instance>
<endpoint>
<args>
<arg name="name">
<title>{data_input_title} Data Input Name</title>
</arg>
{param_str}
</args>
</endpoint>
</scheme>
""".format(
single_instance=(str(single_instance)).lower(),
data_input_title=mod_input_name,
param_str=''.join(param_string_list),
description=description,
)
def _setup_signal_handler(data_loader, ta_short_name):
"""
Setup signal handlers
:data_loader: data_loader.DataLoader instance
"""
def _handle_exit(signum, frame):
stulog.logger.info("{} receives exit signal".format(ta_short_name))
if data_loader is not None:
data_loader.tear_down()
utils.handle_tear_down_signals(_handle_exit)
def _handle_file_changes(data_loader):
"""
:reload conf files and exit
"""
def _handle_refresh(changed_files):
stulog.logger.info("Detect {} changed, reboot itself".format(
changed_files))
data_loader.tear_down()
return _handle_refresh
def _get_conf_files(settings):
rest_root = settings.get("meta").get("restRoot")
file_list = [rest_root + "_settings.conf"]
if settings.get("pages") and settings.get("pages").get("configuration"):
configs = settings.get("pages").get("configuration")
tabs = configs.get("tabs") if configs.get("tabs") else []
for tab in tabs:
if tab.get("table"):
file_list.append(rest_root + "_" + tab.get("name") + ".conf")
ta_dir = get_app_root_dir()
return [op.join(ta_dir, "local", f) for f in file_list]
def run(collector_cls, settings, checkpoint_cls=None, config_cls=None,
log_suffix=None, single_instance=True, cc_json_file=None):
"""
Main loop. Run this TA forever
"""
ta_short_name = settings["meta"]["name"].lower()
# This is for stdout flush
utils.disable_stdout_buffer()
# http://bugs.python.org/issue7980
time.strptime('2016-01-01', '%Y-%m-%d')
loader = dl.create_data_loader()
# handle signal
_setup_signal_handler(loader, ta_short_name)
# monitor files to reboot
try:
monitor = fm.FileMonitor(_handle_file_changes(loader),
_get_conf_files(settings))
loader.add_timer(monitor.check_changes, time.time(), 10)
except Exception:
stulog.logger.exception("Fail to add files for monitoring")
# add orphan process handling, which will check each 1 second
orphan_checker = opm.OrphanProcessChecker(loader.tear_down)
loader.add_timer(orphan_checker.check_orphan, time.time(), 1)
tconfig = tc.create_ta_config(settings, config_cls or tc.TaConfig,
log_suffix, single_instance=single_instance)
task_configs = tconfig.get_task_configs()
if not task_configs:
stulog.logger.debug("No task and exiting...")
return
meta_config = tconfig.get_meta_config()
meta_config["cc_json_file"] = cc_json_file
if tconfig.is_shc_member():
# Don't support SHC env
stulog.logger.error("This host is in search head cluster environment , "
"will exit.")
return
# In this case, use file for checkpoint
if _is_checkpoint_dir_length_exceed_limit(tconfig,
meta_config["checkpoint_dir"]):
stulog.logger.error("The length of the checkpoint directory path: '{}' "
"is too long. The max length we support is {}",
meta_config["checkpoint_dir"],
__CHECKPOINT_DIR_MAX_LEN__)
return
jobs = [
tdc.create_data_collector(
loader,
tconfig,
meta_config,
task_config,
collector_cls,
checkpoint_cls=checkpoint_cls or cpmgr.TACheckPointMgr
)
for task_config in task_configs
]
loader.run(jobs)
def _is_checkpoint_dir_length_exceed_limit(config, checkpoint_dir):
return platform.system() == 'Windows' \
and not config.is_search_head() \
and len(checkpoint_dir) >= __CHECKPOINT_DIR_MAX_LEN__
def validate_config():
"""
Validate inputs.conf
"""
_, configs = modinput.get_modinput_configs_from_stdin()
return 0
def usage():
"""
Print usage of this binary
"""
hlp = "%s --scheme|--validate-arguments|-h"
print >> sys.stderr, hlp % sys.argv[0]
sys.exit(1)
def main(
collector_cls,
schema_file_path,
log_suffix="modinput",
checkpoint_cls=None,
config_cls=None,
cc_json_file=None,
schema_para_list=None,
single_instance=True
):
"""
Main entry point
"""
assert collector_cls, "ucc modinput collector is None."
assert schema_file_path, "ucc modinput schema file is None"
settings = ld(schema_file_path)
mod_input_name = get_mod_input_script_name()
args = sys.argv
if len(args) > 1:
if args[1] == "--scheme":
do_scheme(
mod_input_name=mod_input_name,
schema_para_list=schema_para_list,
single_instance=single_instance
)
elif args[1] == "--validate-arguments":
sys.exit(validate_config())
elif args[1] in ("-h", "--h", "--help"):
usage()
else:
usage()
else:
try:
run(
collector_cls,
settings,
checkpoint_cls=checkpoint_cls,
config_cls=config_cls,
log_suffix=log_suffix,
single_instance=single_instance,
cc_json_file=cc_json_file
)
except Exception:
stulog.logger.exception(
"{} task encounter exception".format(mod_input_name))
stulog.logger.info("End {} task".format(mod_input_name))
sys.exit(0)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/splunk_ta_import_declare.py
-17
View File
@@ -1,17 +0,0 @@
"""
This module is used to filter and reload PATH.
"""
import os
import sys
import re
ta_name = os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
ta_lib_name = re.sub("[^\w]+", "_", ta_name.lower())
assert ta_name or ta_name == "package", "TA name is None or package"
pattern = re.compile(r"[\\/]etc[\\/]apps[\\/][^\\/]+[\\/]bin[\\/]?$")
new_paths = [path for path in sys.path if not pattern.search(path) or ta_name in path]
new_paths.insert(0, os.path.sep.join([os.path.dirname(__file__), ta_lib_name]))
sys.path = new_paths
package/bin/da_ess_contentupdate/cloudconnectlib/splunktacollectorlib/ta_cloud_connect_client.py
-41
View File
@@ -1,41 +0,0 @@
from .data_collection.ta_data_client import TaDataClient
from ..splunktacollectorlib.common import log as stulog
from ..splunktacollectorlib.data_collection import ta_consts as c
from ..common.log import set_cc_logger
class TACloudConnectClient(TaDataClient):
def __init__(self,
meta_config,
task_config,
checkpoint_mgr=None,
event_writer=None
):
super(TACloudConnectClient, self).__init__(meta_config,
task_config,
checkpoint_mgr,
event_writer)
self._set_log()
self._cc_config_file = self._meta_config["cc_json_file"]
from ..core.pipemgr import PipeManager
from ..client import CloudConnectClient as Client
self._pipe_mgr = PipeManager(event_writer=event_writer)
self._client = Client(self._task_config, self._cc_config_file,
checkpoint_mgr)
def _set_log(self):
pairs = ['{}="{}"'.format(c.stanza_name, self._task_config[
c.stanza_name])]
set_cc_logger(stulog.logger,
logger_prefix="[{}]".format(" ".join(pairs)))
def is_stopped(self):
return self._stop
def stop(self):
self._stop = True
self._client.stop()
def get(self):
self._client.start()
raise StopIteration
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/__init__.py
-2
View File
@@ -1,2 +0,0 @@
__version__ = "0.9"
__license__ = "Splunk"
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/__init__.py
View File
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/consts.py
-1
View File
@@ -1 +0,0 @@
util_log = "util"
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/log.py
-137
View File
@@ -1,137 +0,0 @@
"""
Copyright (C) 2005-2015 Splunk Inc. All Rights Reserved.
log utility for TA
"""
import logging
import logging.handlers as handlers
import os
import os.path as op
from ..splunk_platform import make_splunkhome_path
from . import util as cutil
from .pattern import singleton
import time
logging.Formatter.converter = time.gmtime
__LOG_FORMAT__ = "%(asctime)s +0000 log_level=%(levelname)s, pid=%(process)d, " \
"tid=%(threadName)s, file=%(filename)s, " \
"func_name=%(funcName)s, code_line_no=%(lineno)d | %(message)s"
def log_enter_exit(logger):
"""
Log decorator to log function enter and exit
"""
def log_decorator(func):
def wrapper(*args, **kwargs):
logger.debug("{} entered.".format(func.__name__))
result = func(*args, **kwargs)
logger.debug("{} exited.".format(func.__name__))
return result
return wrapper
return log_decorator
def check_add_stderr_handler():
env_var = os.environ.get('splunk.cloudconnect.settings.logging.type')
return env_var and env_var == "stderr"
@singleton
class Logs(object):
def __init__(self, namespace=None, default_level=logging.INFO):
self._loggers = {}
self._default_level = default_level
if namespace is None:
namespace = cutil.get_appname_from_path(op.abspath(__file__))
if namespace:
namespace = namespace.lower()
self._namespace = namespace
def get_logger(self, name, level=None,
maxBytes=25000000, backupCount=5):
"""
Set up a default logger.
:param name: The log file name.
:param level: The logging level.
:param maxBytes: The maximum log file size before rollover.
:param backupCount: The number of log files to retain.
"""
# Strip ".py" from the log file name if auto-generated by a script.
if level is None:
level = self._default_level
name = self._get_log_name(name)
if name in self._loggers:
return self._loggers[name]
logger = logging.getLogger(name)
if check_add_stderr_handler():
import sys
ch = logging.StreamHandler(sys.stderr)
ch.setLevel(logging.ERROR)
formatter = logging.Formatter(__LOG_FORMAT__)
ch.setFormatter(formatter)
logger.addHandler(ch)
else:
logfile = make_splunkhome_path(["var", "log", "splunk", name])
handler_exists = any(
[True for h in logger.handlers if h.baseFilename == logfile])
if not handler_exists:
file_handler = handlers.RotatingFileHandler(
logfile, mode="a", maxBytes=maxBytes, backupCount=backupCount)
formatter = logging.Formatter(__LOG_FORMAT__ )
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel(level)
logger.propagate = False
self._loggers[name] = logger
return logger
def set_level(self, level, name=None):
"""
Change the log level of the logging
:param level: the level of the logging to be setLevel
:param name: the name of the logging to set, in case it is not set,
all the loggers will be affected
"""
if name is not None:
name = self._get_log_name(name)
logger = self._loggers.get(name)
if logger is not None:
logger.setLevel(level)
else:
self._default_level = level
for logger in self._loggers.itervalues():
logger.setLevel(level)
def _get_log_name(self, name):
if name.endswith(".py"):
name = name.replace(".py", "")
if self._namespace:
name = "{}_{}.log".format(self._namespace, name)
else:
name = "{}.log" .format(name)
return name
# Global logger
logger = Logs().get_logger("util")
def reset_logger(name):
"""
Reset global logger.
"""
global logger
logger = Logs().get_logger(name)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/pattern.py
-37
View File
@@ -1,37 +0,0 @@
"""
Copyright (C) 2005-2015 Splunk Inc. All Rights Reserved.
Commonly used design partten for python user, includes:
- singleton (Decorator function used to build singleton)
"""
from functools import wraps
def singleton(class_):
"""
Singleton decoorator function.
"""
instances = {}
@wraps(class_)
def getinstance(*args, **kwargs):
if class_ not in instances:
instances[class_] = class_(*args, **kwargs)
return instances[class_]
return getinstance
class Singleton(type):
"""
Singleton meta class
"""
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(
*args, **kwargs)
print cls
return cls._instances[cls]
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/util.py
-108
View File
@@ -1,108 +0,0 @@
"""
Copyright (C) 2005-2015 Splunk Inc. All Rights Reserved.
"""
import os
import os.path as op
import datetime
import sys
import gc
import urllib
def handle_tear_down_signals(callback):
import signal
signal.signal(signal.SIGTERM, callback)
signal.signal(signal.SIGINT, callback)
if os.name == "nt":
signal.signal(signal.SIGBREAK, callback)
def datetime_to_seconds(dt):
epoch_time = datetime.datetime.utcfromtimestamp(0)
return (dt - epoch_time).total_seconds()
def is_true(val):
value = str(val).strip().upper()
if value in ("1", "TRUE", "T", "Y", "YES"):
return True
return False
def is_false(val):
value = str(val).strip().upper()
if value in ("0", "FALSE", "F", "N", "NO", "NONE", ""):
return True
return False
def remove_http_proxy_env_vars():
for k in ("http_proxy", "https_proxy"):
if k in os.environ:
del os.environ[k]
elif k.upper() in os.environ:
del os.environ[k.upper()]
def get_appname_from_path(absolute_path):
absolute_path = op.normpath(absolute_path)
parts = absolute_path.split(os.path.sep)
parts.reverse()
for key in ("apps", "slave-apps", "master-apps"):
try:
idx = parts.index(key)
except ValueError:
continue
else:
try:
if parts[idx + 1] == "etc":
return parts[idx - 1]
except IndexError:
pass
continue
#return None
return "-"
def escape_cdata(data):
# FIXME: This is a workaround for JIRA [addon-10459]
data = data.decode("utf-8", errors="replace").encode("utf-8", errors="xmlcharrefreplace")
data = data.replace("]]>", "]]&gt;")
if data.endswith("]"):
data = data[:-1] + "%5D"
return data
def extract_datainput_name(stanza_name):
"""
stansa_name: string like aws_s3://my_s3_data_input
"""
sep = "://"
try:
idx = stanza_name.index(sep)
except ValueError:
return stanza_name
return stanza_name[idx + len(sep):]
def escape_json_control_chars(json_str):
control_chars = ((r"\n", "\\\\n"), (r"\r", "\\\\r"),
(r"\r\n", "\\\\r\\\\n"))
for ch, replace in control_chars:
json_str = json_str.replace(ch, replace)
return json_str
def disable_stdout_buffer():
os.environ["PYTHONUNBUFFERED"] = "1"
sys.stdout = os.fdopen(sys.stdout.fileno(), "w", 0)
gc.garbage.append(sys.stdout)
def format_stanza_name(name):
return urllib.quote(name.encode("utf-8"), "")
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/common/xml_dom_parser.py
-47
View File
@@ -1,47 +0,0 @@
import re
from xml.etree import cElementTree as et
def parse_conf_xml_dom(xml_content):
"""
@xml_content: XML DOM from splunkd
"""
m = re.search(r'xmlns="([^"]+)"', xml_content)
ns = m.group(1)
m = re.search(r'xmlns:s="([^"]+)"', xml_content)
sub_ns = m.group(1)
entry_path = "./{%s}entry" % ns
stanza_path = "./{%s}title" % ns
key_path = "./{%s}content/{%s}dict/{%s}key" % (ns, sub_ns, sub_ns)
meta_path = "./{%s}dict/{%s}key" % (sub_ns, sub_ns)
list_path = "./{%s}list/{%s}item" % (sub_ns, sub_ns)
xml_conf = et.fromstring(xml_content)
stanza_objs = []
for entry in xml_conf.iterfind(entry_path):
for stanza in entry.iterfind(stanza_path):
stanza_obj = {"name": stanza.text,"stanza": stanza.text}
break
else:
continue
for key in entry.iterfind(key_path):
if key.get("name") == "eai:acl":
meta = {}
for k in key.iterfind(meta_path):
meta[k.get("name")] = k.text
stanza_obj[key.get("name")] = meta
elif key.get("name") != "eai:attributes":
name = key.get("name")
if name.startswith("eai:"):
name = name[4:]
list_vals = [k.text for k in key.iterfind(list_path)]
if list_vals:
stanza_obj[name] = list_vals
else:
stanza_obj[name] = key.text
if key.text == "None":
stanza_obj[name] = None
stanza_objs.append(stanza_obj)
return stanza_objs
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/concurrent/__init__.py
View File
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/concurrent/concurrent_executor.py
-88
View File
@@ -1,88 +0,0 @@
"""
Concurrent executor provides concurrent executing function either in
a thread pool or a process pool
"""
from ..concurrent import thread_pool as tp
from ..concurrent import process_pool as pp
class ConcurrentExecutor(object):
def __init__(self, config):
"""
:param config: dict like object, contains thread_min_size (int),
thread_max_size (int), daemonize_thread (bool),
process_size (int)
"""
self._io_executor = tp.ThreadPool(config.get("thread_min_size", 0),
config.get("thread_max_size", 0),
config.get("task_queue_size", 1024),
config.get("daemonize_thread", True))
self._compute_executor = None
if config.get("process_size", 0):
self._compute_executor = pp.ProcessPool(
config.get("process_size", 0))
def start(self):
self._io_executor.start()
def tear_down(self):
self._io_executor.tear_down()
if self._compute_executor is not None:
self._compute_executor.tear_down()
def run_io_func_sync(self, func, args=(), kwargs=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:return whatever the func returns
"""
return self._io_executor.apply(func, args, kwargs)
def run_io_func_async(self, func, args=(), kwargs=None, callback=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:calllback: when func is done and without exception, call the callback
:return whatever the func returns
"""
return self._io_executor.apply_async(func, args, kwargs, callback)
def enqueue_io_funcs(self, funcs, block=True):
"""
run jobs in a fire and forget way, no result will be handled
over to clients
:param funcs: tuple/list-like or generator like object, func shall be
callable
"""
return self._io_executor.enqueue_funcs(funcs, block)
def run_compute_func_sync(self, func, args=(), kwargs={}):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:return whatever the func returns
"""
assert self._compute_executor is not None
return self._compute_executor.apply(func, args, kwargs)
def run_compute_func_async(self, func, args=(), kwargs={}, callback=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:calllback: when func is done and without exception, call the callback
:return whatever the func returns
"""
assert self._compute_executor is not None
return self._compute_executor.apply_async(func, args, kwargs, callback)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/concurrent/process_pool.py
-63
View File
@@ -1,63 +0,0 @@
"""
A wrapper of multiprocessing.pool
"""
import multiprocessing
from ..common import log
class ProcessPool(object):
"""
A simple wrapper of multiprocessing.pool
"""
def __init__(self, size=0, maxtasksperchild=10000):
if size <= 0:
size = multiprocessing.cpu_count()
self.size = size
self._pool = multiprocessing.Pool(processes=size,
maxtasksperchild=maxtasksperchild)
self._stopped = False
def tear_down(self):
"""
Tear down the pool
"""
if self._stopped:
log.logger.info("ProcessPool has already stopped.")
return
self._stopped = True
self._pool.close()
self._pool.join()
log.logger.info("ProcessPool stopped.")
def apply(self, func, args=(), kwargs={}):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:return whatever the func returns
"""
if self._stopped:
log.logger.info("ProcessPool has already stopped.")
return None
return self._pool.apply(func, args, kwargs)
def apply_async(self, func, args=(), kwargs={}, callback=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:callback: when func is done without exception, call this callack
:return whatever the func returns
"""
if self._stopped:
log.logger.info("ProcessPool has already stopped.")
return None
return self._pool.apply_async(func, args, kwargs, callback)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/concurrent/thread_pool.py
-344
View File
@@ -1,344 +0,0 @@
"""
A simple thread pool implementation
"""
import threading
import Queue
import multiprocessing
import traceback
import exceptions
from time import time
from ..common import log
class ThreadPool(object):
"""
A simple thread pool implementation
"""
_high_watermark = 0.2
_resize_window = 10
def __init__(self, min_size=1, max_size=128,
task_queue_size=1024, daemon=True):
assert task_queue_size
if not min_size or min_size <= 0:
min_size = multiprocessing.cpu_count()
if not max_size or max_size <= 0:
max_size = multiprocessing.cpu_count() * 8
self._min_size = min_size
self._max_size = max_size
self._daemon = daemon
self._work_queue = Queue.Queue(task_queue_size)
self._thrs = []
for _ in range(min_size):
thr = threading.Thread(target=self._run)
self._thrs.append(thr)
self._admin_queue = Queue.Queue()
self._admin_thr = threading.Thread(target=self._do_admin)
self._last_resize_time = time()
self._last_size = min_size
self._lock = threading.Lock()
self._occupied_threads = 0
self._count_lock = threading.Lock()
self._started = False
def start(self):
"""
Start threads in the pool
"""
with self._lock:
if self._started:
return
self._started = True
for thr in self._thrs:
thr.daemon = self._daemon
thr.start()
self._admin_thr.start()
log.logger.info("ThreadPool started.")
def tear_down(self):
"""
Tear down thread pool
"""
with self._lock:
if not self._started:
return
self._started = False
for thr in self._thrs:
self._work_queue.put(None, block=True)
self._admin_queue.put(None)
if not self._daemon:
log.logger.info("Wait for threads to stop.")
for thr in self._thrs:
thr.join()
self._admin_thr.join()
log.logger.info("ThreadPool stopped.")
def enqueue_funcs(self, funcs, block=True):
"""
run jobs in a fire and forget way, no result will be handled
over to clients
:param funcs: tuple/list-like or generator like object, func shall be
callable
"""
if not self._started:
log.logger.info("ThreadPool has already stopped.")
return
for func in funcs:
self._work_queue.put(func, block)
def apply_async(self, func, args=(), kwargs=None, callback=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:callback: when func is done and without exception, call the callback
:return AsyncResult, clients can poll or wait the result through it
"""
if not self._started:
log.logger.info("ThreadPool has already stopped.")
return None
res = AsyncResult(func, args, kwargs, callback)
self._work_queue.put(res)
return res
def apply(self, func, args=(), kwargs=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:return whatever the func returns
"""
if not self._started:
log.logger.info("ThreadPool has already stopped.")
return None
res = self.apply_async(func, args, kwargs)
return res.get()
def size(self):
return self._last_size
def resize(self, new_size):
"""
Resize the pool size, spawn or destroy threads if necessary
"""
if new_size <= 0:
return
if self._lock.locked() or not self._started:
log.logger.info("Try to resize thread pool during the tear "
"down process, do nothing")
return
with self._lock:
self._remove_exited_threads_with_lock()
size = self._last_size
self._last_size = new_size
if new_size > size:
for _ in xrange(new_size - size):
thr = threading.Thread(target=self._run)
thr.daemon = self._daemon
thr.start()
self._thrs.append(thr)
elif new_size < size:
for _ in xrange(size - new_size):
self._work_queue.put(None)
log.logger.info("Finished ThreadPool resizing. New size=%d", new_size)
def _remove_exited_threads_with_lock(self):
"""
Join the exited threads last time when resize was called
"""
joined_thrs = set()
for thr in self._thrs:
if not thr.is_alive():
try:
if not thr.daemon:
thr.join(timeout=0.5)
joined_thrs.add(thr.ident)
except RuntimeError:
pass
if joined_thrs:
live_thrs = []
for thr in self._thrs:
if thr.ident not in joined_thrs:
live_thrs.append(thr)
self._thrs = live_thrs
def _do_resize_according_to_loads(self):
if (self._last_resize_time and
time() - self._last_resize_time < self._resize_window):
return
thr_size = self._last_size
free_thrs = thr_size - self._occupied_threads
work_size = self._work_queue.qsize()
log.logger.debug("current_thr_size=%s, free_thrs=%s, work_size=%s",
thr_size, free_thrs, work_size)
if work_size and work_size > free_thrs:
if thr_size < self._max_size:
thr_size = min(thr_size * 2, self._max_size)
self.resize(thr_size)
elif free_thrs > 0:
free = free_thrs * 1.0
if free / thr_size >= self._high_watermark and free_thrs >= 2:
# 20 % thrs are idle, tear down half of the idle ones
thr_size = thr_size - free_thrs / 2
if thr_size > self._min_size:
self.resize(thr_size)
self._last_resize_time = time()
def _do_admin(self):
admin_q = self._admin_queue
resize_win = self._resize_window
while 1:
try:
wakup = admin_q.get(timeout=resize_win + 1)
except Queue.Empty:
self._do_resize_according_to_loads()
continue
if wakup is None:
break
else:
self._do_resize_according_to_loads()
log.logger.info("ThreadPool admin thread=%s stopped.",
threading.current_thread().getName())
def _run(self):
"""
Threads callback func, run forever to handle jobs from the job queue
"""
work_queue = self._work_queue
count_lock = self._count_lock
while 1:
log.logger.debug("Going to get job")
func = work_queue.get()
if func is None:
break
if not self._started:
break
log.logger.debug("Going to exec job")
with count_lock:
self._occupied_threads += 1
try:
func()
except Exception:
log.logger.error(traceback.format_exc())
with count_lock:
self._occupied_threads -= 1
log.logger.debug("Done with exec job")
log.logger.info("Thread work_queue_size=%d", work_queue.qsize())
log.logger.debug("Worker thread %s stopped.",
threading.current_thread().getName())
class AsyncResult(object):
def __init__(self, func, args, kwargs, callback):
self._func = func
self._args = args
self._kwargs = kwargs
self._callback = callback
self._q = Queue.Queue()
def __call__(self):
try:
if self._args and self._kwargs:
res = self._func(*self._args, **self._kwargs)
elif self._args:
res = self._func(*self._args)
elif self._kwargs:
res = self._func(**self._kwargs)
else:
res = self._func()
except Exception as e:
self._q.put(e)
return
else:
self._q.put(res)
if self._callback is not None:
self._callback()
def get(self, timeout=None):
"""
Return the result when it arrives. If timeout is not None and the
result does not arrive within timeout seconds then
multiprocessing.TimeoutError is raised. If the remote call raised an
exception then that exception will be reraised by get().
"""
try:
res = self._q.get(timeout=timeout)
except Queue.Empty:
raise multiprocessing.TimeoutError("Timed out")
if isinstance(res, Exception):
raise res
return res
def wait(self, timeout=None):
"""
Wait until the result is available or until timeout seconds pass.
"""
try:
res = self._q.get(timeout=timeout)
except Queue.Empty:
pass
else:
self._q.put(res)
def ready(self):
"""
Return whether the call has completed.
"""
return len(self._q)
def successful(self):
"""
Return whether the call completed without raising an exception.
Will raise AssertionError if the result is not ready.
"""
if not self.ready():
raise exceptions.AssertionError("Function is not ready")
res = self._q.get()
self._q.put(res)
if isinstance(res, Exception):
return False
return True
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/__init__.py
View File
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/conf_endpoints.py
-142
View File
@@ -1,142 +0,0 @@
from .request import content_request
from ..common import util
from ..common import xml_dom_parser as xdp
CONF_ENDPOINT = "%s/servicesNS/%s/%s/configs/conf-%s"
def _conf_endpoint_ns(uri, owner, app, conf_name):
return CONF_ENDPOINT % (uri, owner, app, conf_name)
def reload_conf(splunkd_uri, session_key, app_name, conf_name, throw=False):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param conf_names: a list of the name of the conf file, e.g. ["props"]
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
"""
uri = _conf_endpoint_ns(splunkd_uri, "nobody", app_name, conf_name)
uri += "/_reload"
msg = "Failed to reload conf in app=%s: %s" % (app_name, conf_name)
try:
content_request(uri, session_key, "GET", None, msg)
except Exception:
if throw:
raise
def create_stanza(splunkd_uri, session_key, owner, app_name, conf_name,
stanza, key_values):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param conf_name: the name of the conf file, e.g. "props"
:param stanza: stanza name, e.g. "aws:cloudtrail"
:param key_values: the key-value dict of the stanza
:return: None on success otherwise throw exception
"""
uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
msg = "Failed to create stanza=%s in conf=%s" % (stanza, conf_name)
payload = {"name": unicode(stanza).encode('utf-8')}
for key in key_values:
if key != "name":
payload[key] = str(key_values[key])
content_request(uri, session_key, "POST", payload, msg)
def get_conf(splunkd_uri, session_key, owner, app_name, conf_name,
stanza=None):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param conf_name: the name of the conf file, e.g. "props"
:param stanza: stanza name, e.g. "aws:cloudtrail"
:return: a list of stanzas in the conf file, including metadata
"""
uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
if stanza:
uri += "/" + util.format_stanza_name(stanza)
# get all the stanzas at one time
uri += "?count=0&offset=0"
msg = "Failed to get stanza=%s in conf=%s" % (stanza if stanza else stanza, conf_name)
content = content_request(uri, session_key, "GET", None, msg)
return xdp.parse_conf_xml_dom(content)
def update_stanza(splunkd_uri, session_key, owner, app_name, conf_name,
stanza, key_values):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param conf_name: the name of the conf file, e.g. "props"
:param stanza: stanza name, e.g. "aws:cloudtrail"
:param key_values: the key-value dict of the stanza
:return: None on success otherwise raise exception
"""
uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
uri += "/" + util.format_stanza_name(stanza)
msg = "Failed to update stanza=%s in conf=%s" % (stanza, conf_name)
return content_request(uri, session_key, "POST", key_values, msg)
def delete_stanza(splunkd_uri, session_key, owner, app_name, conf_name,
stanza, throw=False):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param conf_name: the name of the conf file, e.g. "props"
:param stanza: stanza name, e.g. "aws:cloudtrail"
:return: None on success otherwise raise exception
"""
uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
uri += "/" + util.format_stanza_name(stanza)
msg = "Failed to delete stanza=%s in conf=%s" % (stanza, conf_name)
content_request(uri, session_key, "DELETE", None, msg)
def stanza_exist(splunkd_uri, session_key, owner, app_name, conf_name,
stanza):
try:
res = get_conf(splunkd_uri, session_key, owner, app_name, conf_name,
stanza)
return len(res) > 0
except Exception:
return False
def operate_conf(splunkd_uri, session_key, owner, app_name, conf_name,
stanza, operation):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param conf_name: the name of the conf file, e.g. "props"
:param stanza: stanza name, e.g. "aws:cloudtrail"
:param operation: must be "disable" or "enable"
"""
assert operation in ("disable", "enable")
uri = _conf_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
uri += "/%s/%s" % (util.format_stanza_name(stanza), operation)
msg = "Failed to disable/enable stanza=%s in conf=%s" % (stanza, conf_name)
content_request(uri, session_key, "POST", None, msg)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/conf_manager.py
-225
View File
@@ -1,225 +0,0 @@
"""
This module hanles configuration related stuff
"""
import os.path as op
from . import conf_endpoints as scmc
from . import data_input_endpoints as scmdi
from . import property_endpoints as scmp
from . import request as req
def conf_file2name(conf_file):
conf_name = op.basename(conf_file)
if conf_name.endswith(".conf"):
conf_name = conf_name[:-5]
return conf_name
class ConfManager(object):
def __init__(self, splunkd_uri, session_key, owner="nobody", app_name="-"):
"""
:app_name: when creating conf stanza, app_name is required to set not
to "-"
:owner: when creating conf stanza, app_name is required to set not
to "-"
"""
self.splunkd_uri = splunkd_uri
self.session_key = session_key
self.owner = owner
self.app_name = app_name
def set_appname(self, appname):
"""
This are cases we need edit/remove/create confs in different app
context. call this interface to switch app context before manipulate
the confs in different app context
"""
self.app_name = appname
def all_stanzas(self, conf_name, do_reload=False, ret_metadata=False):
"""
:return: a list of dict stanza objects if successful.
Otherwise raise exception
"""
if do_reload:
self.reload_conf(conf_name)
stanzas = scmc.get_conf(self.splunkd_uri, self.session_key,
"-", "-", conf_name)
return self._delete_metadata(stanzas, ret_metadata)
def all_stanzas_as_dicts(self, conf_name, do_reload=False,
ret_metadata=False):
"""
:return: a dict of dict stanza objects if successful.
otherwise raise exception
"""
stanzas = self.all_stanzas(conf_name, do_reload, ret_metadata)
return {stanza["name"]: stanza for stanza in stanzas}
def get_stanza(self, conf_name, stanza,
do_reload=False, ret_metadata=False):
"""
@return dict if success otherwise raise exception
"""
if do_reload:
self.reload_conf(conf_name)
stanzas = scmc.get_conf(self.splunkd_uri, self.session_key,
"-", "-", conf_name, stanza)
stanzas = self._delete_metadata(stanzas, ret_metadata)
return stanzas[0]
def reload_conf(self, conf_name):
scmc.reload_conf(self.splunkd_uri, self.session_key, "-", conf_name)
def enable_conf(self, conf_name, stanza):
scmc.operate_conf(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
conf_name, stanza, "enable")
def disable_conf(self, conf_name, stanza):
scmc.operate_conf(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
conf_name, stanza, "disable")
def get_property(self, conf_name, stanza, key, do_reload=False):
if do_reload:
self.reload_conf(conf_name)
return scmp.get_property(self.splunkd_uri, self.session_key,
"-", "-", conf_name, stanza, key)
def stanza_exist(self, conf_name, stanza):
return scmc.stanza_exist(self.splunkd_uri, self.session_key,
"-", "-", conf_name, stanza)
def create_stanza(self, conf_name, stanza, key_values):
scmc.create_stanza(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
conf_name, stanza, key_values)
def update_stanza(self, conf_name, stanza, key_values):
scmc.update_stanza(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
conf_name, stanza, key_values)
def delete_stanza(self, conf_name, stanza):
scmc.delete_stanza(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
conf_name, stanza)
def create_properties(self, conf_name, stanza):
scmp.create_properties(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
conf_name, stanza)
def update_properties(self, conf_name, stanza, key_values):
scmp.update_properties(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
conf_name, stanza, key_values)
def delete_stanzas(self, conf_name, stanzas):
"""
:param stanzas: list of stanzas
:return: list of failed stanzas
"""
failed_stanzas = []
for stanza in stanzas:
try:
self.delete_stanza(conf_name, stanza)
except Exception:
failed_stanzas.append(stanza)
return failed_stanzas
# data input management
def create_data_input(self, input_type, name, key_values=None):
scmdi.create_data_input(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
input_type, name, key_values)
def update_data_input(self, input_type, name, key_values):
scmdi.update_data_input(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
input_type, name, key_values)
def delete_data_input(self, input_type, name):
scmdi.delete_data_input(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
input_type, name)
def get_data_input(self, input_type, name=None, do_reload=False):
if do_reload:
self.reload_data_input(input_type)
return scmdi.get_data_input(self.splunkd_uri, self.session_key,
"-", "-", input_type, name)
def reload_data_input(self, input_type):
scmdi.reload_data_input(self.splunkd_uri, self.session_key,
"-", "-", input_type)
def enable_data_input(self, input_type, name):
scmdi.operate_data_input(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
input_type, name, "enable")
def disable_data_input(self, input_type, name):
scmdi.operate_data_input(self.splunkd_uri, self.session_key,
self.owner, self.app_name,
input_type, name, "disable")
def data_input_exist(self, input_type, name):
try:
result = self.get_data_input(input_type, name)
except req.ConfNotExistsException:
return False
return result is not None
def all_data_input_stanzas(self, input_type, do_reload=False,
ret_metadata=False):
stanzas = self.get_data_input(input_type, do_reload=do_reload)
for stanza in stanzas:
if "eai:acl" in stanza and "app" in stanza["eai:acl"]:
stanza["appName"] = stanza["eai:acl"]["app"]
stanza["userName"] = stanza["eai:acl"].get("owner", "nobody")
return self._delete_metadata(stanzas, ret_metadata)
def get_data_input_stanza(self, input_type, name, do_reload=False,
ret_metadata=False):
stanzas = self.get_data_input(input_type, name, do_reload)
stanzas = self._delete_metadata(stanzas, ret_metadata)
return stanzas[0]
def delete_data_input_stanzas(self, input_type, names):
"""
:param stanzas: list of stanzas
:return: list of failed stanzas
"""
failed_names = []
for name in names:
try:
self.delete_data_input(input_type, name)
except Exception:
failed_names.append(name)
return failed_names
def _delete_metadata(self, stanzas, ret_metadata):
if stanzas and not ret_metadata:
for stanza in stanzas:
for key in stanza.keys():
if key.startswith("eai:"):
del stanza[key]
return stanzas
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/data_input_endpoints.py
-152
View File
@@ -1,152 +0,0 @@
from .request import content_request
from ..common import util
from ..common import xml_dom_parser as xdp
INPUT_ENDPOINT = "%s/servicesNS/%s/%s/data/inputs/%s"
def _input_endpoint_ns(uri, owner, app, input_type):
return INPUT_ENDPOINT % (uri, owner, app, input_type)
def reload_data_input(splunkd_uri, session_key, owner, app_name,
input_type, throw=False):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param input_type: name of the input type.
if it is a script input, the input is "script",
for modinput, say snow, the input is "snow"
"""
uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
uri += "/_reload"
msg = "Failed to reload data input in app=%s: %s" % (app_name, input_type)
try:
content_request(uri, session_key, "GET", None, msg)
except Exception:
if throw:
raise
def create_data_input(splunkd_uri, session_key, owner, app_name, input_type,
name, key_values):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param input_type: name of the input type.
if it is a script input, the input is "script",
for modinput, say snow, the input is "snow"
:param name: The name of the input stanza to create.
i.e. stanza [<input_type>://<name>] will be created.
:param key_values: a K-V dict of details in the data input stanza.
:return: None on success else raise exception
"""
key_values["name"] = unicode(name).encode('utf-8')
uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
msg = "Failed to create data input in app=%s: %s://%s" % (
app_name, input_type, name)
content_request(uri, session_key, "POST", key_values, msg)
def get_data_input(splunkd_uri, session_key, owner, app_name, input_type,
name=None):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param input_type: name of the input type.
if it is a script input, the input is "script",
for modinput, say snow, the input is "snow"
:param name: The name of the input stanza to create.
i.e. stanza [<input_type>://<name>] will be deleted.
:return: a list of stanzas in the input type, including metadata
"""
uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
if name:
uri += "/" + util.format_stanza_name(name)
# get all the stanzas at one time
uri += "?count=0&offset=0"
msg = "Failed to get data input in app=%s: %s://%s" % (
app_name, input_type, name if name else name)
content = content_request(uri, session_key, "GET", None, msg)
return xdp.parse_conf_xml_dom(content)
def update_data_input(splunkd_uri, session_key, owner, app_name, input_type,
name, key_values):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param input_type: name of the input type.
if it is a script input, the input is "script",
for modinput, say snow, the input is "snow"
:param name: The name of the input stanza to create.
i.e. stanza [<input_type>://<name>] will be updated.
:param key_values: a K-V dict of details in the data input stanza.
:return: raise exception when failure
"""
if "name" in key_values:
del key_values["name"]
uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
uri += "/" + util.format_stanza_name(name)
msg = "Failed to update data input in app=%s: %s://%s" % (
app_name, input_type, name)
content_request(uri, session_key, "POST", key_values, msg)
def delete_data_input(splunkd_uri, session_key, owner, app_name, input_type,
name):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param input_type: name of the input type.
if it is a script input, the input is "script",
for modinput, say snow, the input is "snow"
:param name: The name of the input stanza to create.
i.e. stanza [<input_type>://<name>] will be deleted.
:return raise exception when failed
"""
uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
uri += "/" + util.format_stanza_name(name)
msg = "Failed to delete data input in app=%s: %s://%s" % (
app_name, input_type, name)
content_request(uri, session_key, "DELETE", None, msg)
def operate_data_input(splunkd_uri, session_key, owner, app_name,
input_type, name, operation):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param input_type: name of the input type.
if it is a script input, the input is "script",
for modinput, say snow, the input is "snow"
:param name: The name of the input stanza to create.
i.e. stanza [<input_type>://<name>] will be operated.
:param operation: must be "disable" or "enable"
"""
assert operation in ("disable", "enable")
uri = _input_endpoint_ns(splunkd_uri, owner, app_name, input_type)
uri += "/%s/%s" % (util.format_stanza_name(name), operation)
msg = "Failed to %s data input in app=%s: %s://%s" % (
operation, app_name, input_type, name)
content_request(uri, session_key, "POST", None, msg)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/knowledge_objects.py
-36
View File
@@ -1,36 +0,0 @@
from . import request as req
from ..common import xml_dom_parser as xdp
class KnowledgeObjectManager(object):
def __init__(self, splunkd_uri, session_key):
self.splunkd_uri = splunkd_uri
self.session_key = session_key
def apps(self):
"""
@return: a list of dict containing apps if successfuly otherwise
otherwise raise exceptions
"""
uri = "{}/services/apps/local?count=0&offset=0".format(
self.splunkd_uri)
apps = self._do_request(uri, "GET", None, "Failed to get apps")
return apps
def indexes(self):
"""
@return: a list of dict containing indexes if successfuly
otherwise raise exceptions
"""
uri = "{}/services/data/indexes/?count=0&offset=0".format(
self.splunkd_uri)
indexes = self._do_request(uri, "GET", None, "Failed to get indexes")
return indexes
def _do_request(self, uri, method, payload, err_msg):
_, content = req.content_request(uri, self.session_key, method,
payload, err_msg)
return xdp.parse_conf_xml_dom(content)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/property_endpoints.py
-78
View File
@@ -1,78 +0,0 @@
from .request import content_request
from ..common import util
PROPERTY_ENDPOINT = "%s/servicesNS/%s/%s/properties/%s"
def _property_endpoint_ns(uri, owner, app, conf_name):
return PROPERTY_ENDPOINT % (uri, owner, app, conf_name)
def create_properties(splunkd_uri, session_key, owner, app_name, conf_name,
stanza):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param conf_name: the name of the conf file, e.g. "props"
:param stanza: stanza name, e.g. "aws:cloudtrail"
:return: None on success else raise exception
"""
uri = _property_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
msg = "Properties: failed to create stanza=%s in conf=%s" % \
(stanza, conf_name)
payload = {"__stanza": stanza}
content_request(uri, session_key, "POST", payload, msg)
def get_property(splunkd_uri, session_key, owner, app_name, conf_name,
stanza, key):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param conf_name: the name of the conf file, e.g. "props"
:param stanza: stanza name, e.g. "aws:cloudtrail"
:param key: the property name
:return: the property value
"""
uri = _property_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
uri += "/%s/%s" % (util.format_stanza_name(stanza), key)
msg = "Properties: failed to get conf=%s, stanza=%s, key=%s" % \
(conf_name, stanza, key)
return content_request(uri, session_key, "GET", None, msg)
def update_properties(splunkd_uri, session_key, owner, app_name, conf_name,
stanza, key_values):
"""
:param splunkd_uri: splunkd uri, e.g. https://127.0.0.1:8089
:param session_key: splunkd session key
:param owner: the owner (ACL user), e.g. "-", "nobody"
:param app_name: the app"s name, e.g. "Splunk_TA_aws"
:param conf_name: the name of the conf file, e.g. "props"
:param stanza: stanza name, e.g. "aws:cloudtrail"
:param key_values: the key-value dict of the stanza
:return: raise exception when failed
"""
uri = _property_endpoint_ns(splunkd_uri, owner, app_name, conf_name)
uri += "/" + util.format_stanza_name(stanza)
msg = "Properties: failed to update conf=%s, stanza=%s" % \
(conf_name, stanza)
has_name = False
if "name" in key_values:
has_name = True
name = key_values["name"]
del key_values["name"]
content_request(uri, session_key, "POST", key_values, msg)
if has_name:
key_values["name"] = name
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/request.py
-44
View File
@@ -1,44 +0,0 @@
from .. import rest
from ..common import log
class ConfRequestException(Exception):
pass
class ConfNotExistsException(ConfRequestException):
pass
class ConfExistsException(ConfRequestException):
pass
def content_request(uri, session_key, method, payload, err_msg):
"""
:return: response content if successful otherwise raise
ConfRequestException
"""
resp, content = rest.splunkd_request(uri, session_key, method,
data=payload, retry=3)
if resp is None and content is None:
return None
if resp.status >= 200 and resp.status <= 204:
return content
else:
msg = "%s, status=%s, reason=%s, detail=%s" % (
err_msg, resp.status, resp.reason, content.decode('utf-8'))
if not (method == "GET" and resp.status == 404):
log.logger.error(msg)
if resp.status == 404:
raise ConfNotExistsException(msg)
if resp.status == 409:
raise ConfExistsException(msg)
else:
if content and "already exists" in content:
raise ConfExistsException(msg)
raise ConfRequestException(msg)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/conf_manager/ta_conf_manager.py
-211
View File
@@ -1,211 +0,0 @@
"""
This module hanles high level TA configuration related stuff
"""
import copy
import os.path as op
from . import conf_manager as conf
from . import request as conf_req
from .. import credentials as cred
from ..common import util as utils
class TAConfManager(object):
encrypted_token = "******"
reserved_keys = ("userName", "appName")
def __init__(self, conf_file, splunkd_uri, session_key, appname=None):
if appname is None:
appname = utils.get_appname_from_path(op.abspath(__file__))
self._conf_file = conf.conf_file2name(conf_file)
self._conf_mgr = conf.ConfManager(splunkd_uri, session_key,
app_name=appname)
self._cred_mgr = cred.CredentialManager(
splunkd_uri, session_key, app=appname,
owner="nobody", realm=appname)
self._keys = None
def set_appname(self, appname):
"""
This are cases we need edit/remove/create confs in different app
context. call this interface to switch app context before manipulate
the confs in different app context
"""
self._conf_mgr.set_appname(appname)
self._cred_mgr.set_appname(appname)
def _delete_reserved_keys(self, stanza):
new_stanza = copy.deepcopy(stanza)
for k in self.reserved_keys:
if k in new_stanza:
del new_stanza[k]
return new_stanza
def create(self, stanza):
"""
@stanza: dick like object
{
"name": xxx,
"k1": v1,
"k2": v2,
...
}
@return exception if failure
"""
stanza = self._delete_reserved_keys(stanza)
encrypted_stanza = self._encrypt(stanza)
self._conf_mgr.create_stanza(self._conf_file,
encrypted_stanza["name"],
encrypted_stanza)
def update(self, stanza):
"""
@stanza: dick like object
{
"name": xxx,
"k1": v1,
"k2": v2,
...
}
@return: exception if failure
"""
if not self._conf_mgr.stanza_exist(self._conf_file, stanza["name"]):
self.create(stanza)
else:
stanza = self._delete_reserved_keys(stanza)
encrypted_stanza = self._encrypt(stanza)
self._conf_mgr.update_properties(
self._conf_file, encrypted_stanza["name"], encrypted_stanza)
def delete(self, stanza_name):
"""
@return: exception if failure
"""
try:
stanza = self._conf_mgr.get_stanza(self._conf_file, stanza_name)
except conf_req.ConfNotExistsException:
return
self._delete_creds(stanza)
self._conf_mgr.delete_stanza(self._conf_file, stanza_name)
def get(self, stanza_name, return_acl=False):
"""
@return: dict object if sucess otherwise raise exception
"""
stanza = self._conf_mgr.get_stanza(self._conf_file, stanza_name,
ret_metadata=return_acl)
stanza = self._decrypt(stanza)
stanza["disabled"] = utils.is_true(stanza.get("disabled"))
return stanza
def all(self, filter_disabled=False, return_acl=True):
"""
@return: a dict of dict objects if success
otherwise exception
"""
results = {}
stanzas = self._conf_mgr.all_stanzas(self._conf_file,
ret_metadata=return_acl)
for stanza in stanzas:
stanza = self._decrypt(stanza)
stanza["disabled"] = utils.is_true(stanza.get("disabled"))
if filter_disabled and stanza["disabled"]:
continue
results[stanza["name"]] = stanza
return results
def reload(self):
self._conf_mgr.reload_conf(self._conf_file)
def set_encrypt_keys(self, keys):
"""
:keys: a list keys of a stanza which need to be encrypted
for example: ["username", "password"]
"""
self._keys = keys
def is_encrypted(self, stanza):
"""
:stanza: dict object
return True if the values of encrypt keys equals self.encrypted_token
otherwise return False
"""
if self._keys is None:
return False
for k in stanza.iterkeys():
if k in self._keys:
if stanza.get(k) == self.encrypted_token:
return True
return False
def _encrypt(self, stanza):
"""
:stanza: if self._keys are in stanza, encrypt the values of the key
and then mask the value to self.encrypted_token
"""
if self._keys is None:
return stanza
stanza_to_be_encrypted = {}
for key in self._keys:
if key in stanza:
stanza_to_be_encrypted[key] = stanza[key]
if stanza_to_be_encrypted:
self._cred_mgr.update({stanza["name"]: stanza_to_be_encrypted})
encrypted_stanza = copy.deepcopy(stanza)
for key in stanza_to_be_encrypted.iterkeys():
encrypted_stanza[key] = self.encrypted_token
return encrypted_stanza
return stanza
def _decrypt(self, stanza):
"""
:stanza: if there are keys in self._keys in stanza and if the values of
the keys are self.encrypted_token, decrypt the value
"""
if self._keys is None:
return stanza
stanza_name = stanza["name"]
clear_password = None
for key in self._keys:
if key in stanza and stanza[key] == self.encrypted_token:
clear_password = self._cred_mgr.get_clear_password(
stanza_name)
break
if clear_password:
for key in self._keys:
if key in clear_password[stanza_name]:
stanza[key] = clear_password[stanza_name][key]
return stanza
def _delete_creds(self, stanza):
"""
:stanza: if there are keys of self._keys and the keys are in stanza,
delete the encrypted creds
"""
if self._keys is None:
return
for key in self._keys:
if key in stanza:
self._cred_mgr.delete(stanza["name"])
break
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/credentials.py
-337
View File
@@ -1,337 +0,0 @@
"""
Handles credentials related stuff
"""
import re
import xml.dom.minidom as xdm
from . import rest
from .common import util
from .common import xml_dom_parser as xdp
# Splunk can only encrypt string when length <=255
SPLUNK_CRED_LEN_LIMIT = 255
class CredException(Exception):
pass
class CredNotFound(CredException):
"""
Credential information not exists
"""
pass
def create_credential_manager(username, password, splunkd_uri,
app, owner, realm):
session_key = CredentialManager.get_session_key(
username, password, splunkd_uri)
return CredentialManager(splunkd_uri, session_key, app, owner, realm)
class CredentialManager(object):
"""
Credential related interfaces
"""
def __init__(self, splunkd_uri, session_key,
app="-", owner="nobody", realm=None):
"""
:app: when creating/upating/deleting app is required
"""
self._app = app
self._splunkd_uri = splunkd_uri
self._owner = owner
self._sep = "``splunk_cred_sep``"
if realm:
self._realm = realm
else:
self._realm = app
self._session_key = session_key
def set_appname(self, app):
"""
This are cases we need edit/remove/create confs in different app
context. call this interface to switch app context before manipulate
the confs in different app context
"""
self._app = app
@staticmethod
def get_session_key(username, password,
splunkd_uri="https://localhost:8089"):
"""
Get session key by using login username and passwrod
:return: session_key if successful, None if failed
"""
eid = "".join((splunkd_uri, "/services/auth/login"))
postargs = {
"username": username,
"password": password,
}
response, content = rest.splunkd_request(
eid, None, method="POST", data=postargs)
if response is None and content is None:
raise CredException("Get session key failed.")
xml_obj = xdm.parseString(content)
session_nodes = xml_obj.getElementsByTagName("sessionKey")
if not session_nodes:
raise CredException("Invalid username or password.")
session_key = session_nodes[0].firstChild.nodeValue
if not session_key:
raise CredException("Get session key failed.")
return session_key
def update(self, stanza):
"""
Update or Create credentials based on the stanza
:stanza: nested dict object. The outlayer keys are stanza name, and
inner dict is user/pass key/value pair to be encrypted
{
"stanza_name": {"tommy": "tommypasswod", "jerry": "jerrypassword"}
}
:return: raise on failure
"""
for name, encr_dict in stanza.items():
encrypts = []
for key, val in encr_dict.items():
encrypts.append(key)
encrypts.append(val)
self._update(name, self._sep.join(encrypts))
def _update(self, name, str_to_encrypt):
"""
Update the string for the name.
:return: raise on failure
"""
self.delete(name)
if len(str_to_encrypt) <= SPLUNK_CRED_LEN_LIMIT:
self._create(name, str_to_encrypt)
return
# split the str_to_encrypt when len > 255
length = SPLUNK_CRED_LEN_LIMIT
i = 0
while length < len(str_to_encrypt) + SPLUNK_CRED_LEN_LIMIT:
curr_str = str_to_encrypt[length - SPLUNK_CRED_LEN_LIMIT:length]
length += SPLUNK_CRED_LEN_LIMIT
stanza_name = self._sep.join((name, str(i)))
self._create(stanza_name, curr_str)
i += 1
def _create(self, name, str_to_encrypt):
"""
Create a new stored credential.
:return: raise on failure
"""
payload = {
"name": name,
"password": str_to_encrypt,
"realm": self._realm,
}
endpoint = self._get_endpoint(name)
resp, content = rest.splunkd_request(endpoint, self._session_key,
method="POST", data=payload)
if not resp or resp.status not in (200, 201, "200", "201"):
raise CredException("Failed to encrypt username {}".format(name))
def delete(self, name, throw=False):
"""
Delete the encrypted entry
"""
try:
self._delete(name, throw=True)
except CredNotFound:
# try to delete the split stanzas
try:
stanzas = self._get_all_passwords()
except Exception:
raise
ent_regx = "%s:(%s%s\d+):" % (self._realm, name, self._sep)
ent_pattern = re.compile(ent_regx)
for stanza in stanzas:
stanza_name = stanza.get("name")
match = ent_pattern.match(stanza_name)
if match:
try:
delete_name = match.group(1)
self._delete(delete_name, throw=True)
except CredNotFound:
pass
except CredException:
raise
except CredException:
raise
def _delete(self, name, throw=False):
"""
Delete the encrypted entry
"""
endpoint = self._get_endpoint(name)
response, content = rest.splunkd_request(
endpoint, self._session_key, method="DELETE")
if response is not None and response.status in (404, "404"):
if throw:
raise CredNotFound(
"Credential stanza not exits - {}".format(name))
elif not response or response.status not in (200, 201, "200", "201"):
if throw:
raise CredException(
"Failed to delete credential stanza {}".format(name))
def get_all_passwords(self):
results = {}
all_stanzas = self._get_all_passwords()
for stanza in all_stanzas:
name = stanza.get("name")
match = re.match(r"(.+){}(\d+)".format(self._sep), name)
if match:
actual_name = match.group(1) + ":"
index = int(match.group(2))
if results.get(actual_name):
exist_stanza = results.get(actual_name)
else:
exist_stanza = stanza
exist_stanza['name'] = actual_name
exist_stanza['username'] = \
exist_stanza['username'].split(self._sep)[0]
exist_stanza['clears'] = {}
exist_stanza['encrs'] = {}
try:
exist_stanza['clears'][index] = stanza.get('clear_password')
exist_stanza['encrs'][index] = stanza.get('encr_password')
except KeyError:
exist_stanza['clears'] = {}
exist_stanza['encrs'] = {}
exist_stanza['clears'][index] = stanza.get('clear_password')
exist_stanza['encrs'][index] = stanza.get('encr_password')
results[actual_name] = exist_stanza
else:
results[name] = stanza
# merge the stanzas by index
for name, stanza in results.items():
field_clear = stanza.get('clears')
field_encr = stanza.get('encrs')
if isinstance(field_clear, dict):
clear_password = ""
encr_password = ""
for index in sorted(field_clear.keys()):
clear_password += field_clear.get(index)
encr_password += field_encr.get(index)
stanza['clear_password'] = clear_password
stanza['encr_password'] = encr_password
del stanza['clears']
del stanza['encrs']
return results.values()
def _get_all_passwords(self):
"""
:return: a list of dict when successful, None when failed.
the dict at least contains
{
"realm": xxx,
"username": yyy,
"clear_password": zzz,
}
"""
endpoint = self._get_endpoint()
response, content = rest.splunkd_request(
endpoint, self._session_key, method="GET")
if response and response.status in (200, 201, "200", "201") and content:
return xdp.parse_conf_xml_dom(content)
raise CredException("Failed to get credentials")
def get_clear_password(self, name=None):
"""
:return: clear password(s)
{
stanza_name: {"user": pass}
}
"""
return self._get_credentials("clear_password", name)
def get_encrypted_password(self, name=None):
"""
:return: encyrpted password(s)
"""
return self._get_credentials("encr_password", name)
def _get_credentials(self, prop, name=None):
"""
:return: clear or encrypted password for specified realm, user
"""
all_stanzas = self.get_all_passwords()
results = {}
for stanza in all_stanzas:
if name and not stanza.get("name").endswith(":" + name + ":"):
continue
if stanza.get("realm") == self._realm:
values = stanza[prop].split(self._sep)
if len(values) % 2 == 1:
continue
result = {values[i]: values[i + 1]
for i in range(0, len(values), 2)}
results[stanza.get("username")] = result
return results
@staticmethod
def _build_name(realm, name):
return util.format_stanza_name(
"".join((CredentialManager._escape_string(realm), ":",
CredentialManager._escape_string(name), ":")))
@staticmethod
def _escape_string(string_to_escape):
r"""
Splunk secure credential storage actually requires a custom style of
escaped string where all the :'s are escaped by a single \.
But don't escape the control : in the stanza name.
"""
return string_to_escape.replace(":", "\\:")
def _get_endpoint(self, name=None, query=False):
app = self._app
owner = self._owner
if query:
app = "-"
owner = "-"
if name:
realm_user = self._build_name(self._realm, name)
rest_endpoint = "{}/servicesNS/{}/{}/storage/passwords/{}".format(
self._splunkd_uri, owner, app, realm_user)
else:
rest_endpoint = "{}/servicesNS/{}/{}/storage/passwords?count=-1" \
"".format(self._splunkd_uri, owner, app)
return rest_endpoint
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/event_writer.py
-84
View File
@@ -1,84 +0,0 @@
import Queue
import multiprocessing
import threading
import sys
from collections import Iterable
from .common import log
class EventWriter(object):
def __init__(self, process_safe=False):
if process_safe:
self._mgr = multiprocessing.Manager()
self._event_queue = self._mgr.Queue(1000)
else:
self._event_queue = Queue.Queue(1000)
self._event_writer = threading.Thread(target=self._do_write_events)
self._event_writer.daemon = True
self._started = False
self._exception = False
def start(self):
if self._started:
return
self._started = True
self._event_writer.start()
log.logger.info("Event writer started.")
def tear_down(self):
if not self._started:
return
self._started = False
self._event_queue.put(None)
self._event_writer.join()
log.logger.info("Event writer stopped.")
def isopen(self):
return self._started and (not self._exception)
def write_events(self, events):
if not self.isopen():
return False
if events is None:
return True
self._event_queue.put(events)
return True
def _do_write_events(self):
event_queue = self._event_queue
write = sys.stdout.write
got_shutdown_signal = False
while 1:
try:
event = event_queue.get(timeout=3)
if event is not None:
if isinstance(event, basestring):
write(event)
elif isinstance(event, Iterable):
for evt in event:
write(evt)
else:
log.logger.info("Event writer got tear down signal")
got_shutdown_signal = True
except Queue.Empty:
# We need drain the queue before shutdown
# timeout means empty for now
if got_shutdown_signal:
log.logger.info("Event writer is going to exit...")
break
else:
continue
except Exception:
log.logger.exception("EventWriter encounter exception which may"
"cause data loss, queue leftsize={"
"}".format(
event_queue.qsize()))
self._exception = True
break
log.logger.info("Event writer stopped, queue leftsize={}".format(
event_queue.qsize()))
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/file_monitor.py
-56
View File
@@ -1,56 +0,0 @@
import os.path as op
import traceback
from .common import log
class FileMonitor(object):
def __init__(self, callback, files):
"""
:files: files to be monidtored with full path
"""
self._callback = callback
self._files = files
self.file_mtimes = {
file_name: None for file_name in self._files
}
for k in self.file_mtimes:
if not op.exists(k):
continue
try:
if not op.exists(k):
continue
self.file_mtimes[k] = op.getmtime(k)
except OSError:
log.logger.error("Getmtime for %s, failed: %s",
k, traceback.format_exc())
def __call__(self):
return self.check_changes()
def check_changes(self):
log.logger.debug("Checking files=%s", self._files)
file_mtimes = self.file_mtimes
changed_files = []
for f, last_mtime in file_mtimes.iteritems():
try:
if not op.exists(f):
continue
current_mtime = op.getmtime(f)
if current_mtime != last_mtime:
file_mtimes[f] = current_mtime
changed_files.append(f)
log.logger.info("Detect %s has changed", f)
except OSError:
pass
if changed_files:
if self._callback:
self._callback(changed_files)
return True
return False
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/kv_client.py
-202
View File
@@ -1,202 +0,0 @@
import re
import json
from xml.etree import cElementTree as et
from . import rest as rest
class KVException(Exception):
pass
class KVAlreadyExists(KVException):
pass
class KVNotExists(KVException):
pass
class KVClient(object):
def __init__(self, splunkd_host, session_key):
self._splunkd_host = splunkd_host
self._session_key = session_key
def create_collection(self, collection, app, owner="nobody"):
"""
:collection: collection name
:return: None if successful otherwise KV exception thrown
"""
assert collection
assert app
uri = self._get_config_endpoint(app, owner)
data = {
"name": collection
}
self._do_request(uri, "POST", data)
def list_collection(self, collection=None, app=None, owner="nobody"):
"""
:collection: collection name. When euqals "None", return all
collections in the system.
:return: a list containing the connection names if successful, throws
KVNotExists if no such colection or other exception if other error
happened
"""
uri = self._get_config_endpoint(app, owner, collection)
content = self._do_request(uri, method="GET")
m = re.search(r'xmlns="([^"]+)"', content)
path = "./entry/title"
if m:
ns = m.group(1)
path = "./{%s}entry/{%s}title" % (ns, ns)
collections = et.fromstring(content)
return [node.text for node in collections.iterfind(path)]
def delete_collection(self, collection, app, owner="nobody"):
"""
:collection: collection name to be deleted
:return: None if successful otherwise throw KVNotExists exception if
the collection doesn't exist in the system or other exception if other
error happened
"""
assert collection
uri = self._get_config_endpoint(app, owner, collection)
self._do_request(uri, method="DELETE")
def insert_collection_data(self, collection, data, app, owner="nobody"):
"""
:collection: collection name
:data: dict like key values to be inserted and attached to
this collection
:return: {"_key": "key_id"} when successful, clients can use this
key to do query/delete/update, throws KV exceptions when failed
"""
assert collection
assert data is not None
assert app
uri = self._get_data_endpoint(app, owner, collection)
key = self._do_request(uri, "POST", data,
content_type="application/json")
return json.loads(key)
def delete_collection_data(self, collection, key_id, app, owner="nobody"):
"""
:collection: collection name
:key_id: key id returned when creation. If None, delete all data
associated with this collection
:return: None if successful otherwise throws KV exception
"""
assert collection
uri = self._get_data_endpoint(app, owner, collection, key_id)
self._do_request(uri, "DELETE", content_type="application/json")
def update_collection_data(self, collection, key_id, data,
app, owner="nobody"):
"""
:collection: collection name
:key_id: key id returned when creation
:return: key id if successful otherwise throws KV exception
"""
assert collection
assert key_id
assert app
uri = self._get_data_endpoint(app, owner, collection, key_id)
k = self._do_request(uri, "POST", data,
content_type="application/json")
return json.loads(k)
def get_collection_data(self, collection, key_id, app, owner="nobody"):
"""
:collection: collection name
:key_id: key id returned when creation. If None, get all data
associated with this collection
:return: when key_id is not None, return key values if
successful. when key_id is None, return a list of key values if
sucessful. Throws KV exception if failure
"""
assert collection
uri = self._get_data_endpoint(app, owner, collection, key_id)
k = self._do_request(uri, "GET")
return json.loads(k)
def _do_request(self, uri, method, data=None,
content_type="application/x-www-form-urlencoded"):
headers = {"Content-Type": content_type}
resp, content = rest.splunkd_request(uri, self._session_key,
method, headers, data)
if resp is None and content is None:
raise KVException("Failed uri={0}, data={1}".format(uri, data))
if resp.status in (200, 201):
return content
elif resp.status == 409:
raise KVAlreadyExists("{0}-{1} already exists".format(uri, data))
elif resp.status == 404:
raise KVNotExists("{0}-{1} not exists".format(uri, data))
else:
raise KVException("Failed to {0} {1}, reason={2}".format(
method, uri, resp.reason))
def _get_config_endpoint(self, app, owner, collection=None):
uri = "{0}/servicesNS/{1}/{2}/storage/collections/config"
return self._do_get_endpoint(app, owner, collection, None, uri)
def _get_data_endpoint(self, app, owner, collection, key_id=None):
uri = "{0}/servicesNS/{1}/{2}/storage/collections/data"
return self._do_get_endpoint(app, owner, collection, key_id, uri)
def _do_get_endpoint(self, app, owner, collection, key_id, uri_template):
if not app:
app = "-"
if not owner:
owner = "-"
uri = uri_template.format(self._splunkd_host, owner, app)
if collection is not None:
uri += "/{0}".format(collection)
if key_id is not None:
uri += "/{0}".format(key_id)
return uri
def create_collection(kv_client, collection, appname):
not_exists = False
try:
res = kv_client.list_collection(collection, appname)
except KVNotExists:
not_exists = True
except Exception:
not_exists = True
if not_exists or not res:
for i in xrange(3):
try:
kv_client.create_collection(collection, appname)
except KVAlreadyExists:
return
except Exception as e:
ex = e
else:
return
else:
raise ex
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/modinput.py
-147
View File
@@ -1,147 +0,0 @@
import sys
import subprocess
import traceback
from . import splunk_platform as sp
from .common import log
def _parse_modinput_configs(root, outer_block, inner_block):
"""
When user splunkd spawns modinput script to do config check or run
<?xml version="1.0" encoding="UTF-8"?>
<input>
<server_host>localhost.localdomain</server_host>
<server_uri>https://127.0.0.1:8089</server_uri>
<session_key>xxxyyyzzz</session_key>
<checkpoint_dir>ckpt_dir</checkpoint_dir>
<configuration>
<stanza name="snow://alm_asset">
<param name="duration">60</param>
<param name="host">localhost.localdomain</param>
<param name="index">snow</param>
<param name="priority">10</param>
</stanza>
...
</configuration>
</input>
When user create an stanza through data input on WebUI
<?xml version="1.0" encoding="UTF-8"?>
<items>
<server_host>localhost.localdomain</server_host>
<server_uri>https://127.0.0.1:8089</server_uri>
<session_key>xxxyyyzzz</session_key>
<checkpoint_dir>ckpt_dir</checkpoint_dir>
<item name="abc">
<param name="duration">60</param>
<param name="exclude"></param>
<param name="host">localhost.localdomain</param>
<param name="index">snow</param>
<param name="priority">10</param>
</item>
</items>
"""
confs = root.getElementsByTagName(outer_block)
if not confs:
log.logger.error("Invalid config, missing %s section", outer_block)
raise Exception("Invalid config, missing %s section".format(
outer_block
))
configs = []
stanzas = confs[0].getElementsByTagName(inner_block)
for stanza in stanzas:
config = {}
stanza_name = stanza.getAttribute("name")
if not stanza_name:
log.logger.error("Invalid config, missing name")
raise Exception("Invalid config, missing name")
config["name"] = stanza_name
params = stanza.getElementsByTagName("param")
for param in params:
name = param.getAttribute("name")
if (name and param.firstChild and
param.firstChild.nodeType == param.firstChild.TEXT_NODE):
config[name] = param.firstChild.data
configs.append(config)
return configs
def parse_modinput_configs(config_str):
"""
@config_str: modinput XML configuration feed by splunkd
@return: meta_config and stanza_config
"""
import xml.dom.minidom as xdm
meta_configs = {
"server_host": None,
"server_uri": None,
"session_key": None,
"checkpoint_dir": None,
}
root = xdm.parseString(config_str)
doc = root.documentElement
for tag in meta_configs.iterkeys():
nodes = doc.getElementsByTagName(tag)
if not nodes:
log.logger.error("Invalid config, missing %s section", tag)
raise Exception("Invalid config, missing %s section", tag)
if (nodes[0].firstChild and
nodes[0].firstChild.nodeType == nodes[0].TEXT_NODE):
meta_configs[tag] = nodes[0].firstChild.data
else:
log.logger.error("Invalid config, expect text ndoe")
raise Exception("Invalid config, expect text ndoe")
if doc.nodeName == "input":
configs = _parse_modinput_configs(doc, "configuration", "stanza")
else:
configs = _parse_modinput_configs(root, "items", "item")
return meta_configs, configs
def get_modinput_configs_from_cli(modinput, modinput_stanza=None):
"""
@modinput: modinput name
@modinput_stanza: modinput stanza name, for multiple instance only
"""
assert modinput
splunkbin = sp.get_splunk_bin()
cli = [splunkbin, "cmd", "splunkd", "print-modinput-config", modinput]
if modinput_stanza:
cli.append(modinput_stanza)
out, err = subprocess.Popen(cli, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if err:
log.logger.error("Failed to get modinput configs with error: %s", err)
return None, None
else:
return parse_modinput_configs(out)
def get_modinput_config_str_from_stdin():
"""
Get modinput from stdin which is feed by splunkd
"""
try:
return sys.stdin.read(5000)
except Exception:
log.logger.error(traceback.format_exc())
raise
def get_modinput_configs_from_stdin():
config_str = get_modinput_config_str_from_stdin()
return parse_modinput_configs(config_str)
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/orphan_process_monitor.py
-65
View File
@@ -1,65 +0,0 @@
import os
import threading
import time
import traceback
from ..splunktalib.common import log
class OrphanProcessChecker(object):
def __init__(self, callback=None):
"""
Only work for Linux platform. On Windows platform, is_orphan is always
False
"""
if os.name == "nt":
self._ppid = 0
else:
self._ppid = os.getppid()
self._callback = callback
def is_orphan(self):
if os.name == "nt":
return False
res = self._ppid != os.getppid()
if res:
log.logger.warn("Process=%s has become orphan", os.getpid())
return res
def check_orphan(self):
res = self.is_orphan()
if res and self._callback:
self._callback()
return res
class OrphanProcessMonitor(object):
def __init__(self, callback):
self._checker = OrphanProcessChecker(callback)
self._thr = threading.Thread(target=self._do_monitor)
self._thr.daemon = True
self._started = False
def start(self):
if self._started:
return
self._started = True
self._thr.start()
def stop(self):
self._started = False
def _do_monitor(self):
while self._started:
try:
res = self._checker.check_orphan()
if res:
break
time.sleep(1)
except Exception:
log.logger.error("Failed to monitor orphan process, reason=%s",
traceback.format_exc())
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/rest.py
-122
View File
@@ -1,122 +0,0 @@
import urllib
import json
from traceback import format_exc
from .common import util as scu
from .common import log as log
from httplib2 import (socks, ProxyInfo, Http)
def splunkd_request(splunkd_uri, session_key, method="GET",
headers=None, data=None, timeout=30, retry=1):
"""
:return: httplib2.Response and content
"""
headers = headers if headers is not None else {}
headers["Authorization"] = "Splunk {0}".format(session_key)
content_type = headers.get("Content-Type")
if not content_type:
content_type = headers.get("content-type")
if not content_type:
content_type = "application/x-www-form-urlencoded"
headers["Content-Type"] = content_type
if data is not None:
if content_type == "application/json":
data = json.dumps(data)
else:
data = urllib.urlencode(data)
http = Http(timeout=timeout, disable_ssl_certificate_validation=True)
msg_temp = "Failed to send rest request=%s, errcode=%s, reason=%s"
resp, content = None, None
for _ in range(retry):
try:
resp, content = http.request(splunkd_uri, method=method,
headers=headers, body=data)
except Exception:
log.logger.error(msg_temp, splunkd_uri, "unknown", format_exc())
else:
if resp.status not in (200, 201):
if not (method == "GET" and resp.status == 404):
log.logger.debug(msg_temp, splunkd_uri, resp.status,
code_to_msg(resp, content))
else:
return resp, content
else:
return resp, content
def code_to_msg(resp, content):
code_msg_tbl = {
400: "Request error. reason={}".format(content),
401: "Authentication failure, invalid access credentials.",
402: "In-use license disables this feature.",
403: "Insufficient permission.",
404: "Requested endpoint does not exist.",
409: "Invalid operation for this endpoint. reason={}".format(content),
500: "Unspecified internal server error. reason={}".format(content),
503: ("Feature is disabled in the configuration file. "
"reason={}".format(content)),
}
return code_msg_tbl.get(resp.status, content)
def build_http_connection(config, timeout=120, disable_ssl_validation=False):
"""
:config: dict like, proxy and account information are in the following
format {
"username": xx,
"password": yy,
"proxy_url": zz,
"proxy_port": aa,
"proxy_username": bb,
"proxy_password": cc,
"proxy_type": http,http_no_tunnel,sock4,sock5,
"proxy_rdns": 0 or 1,
}
:return: Http2.Http object
"""
proxy_type_to_code = {
"http": socks.PROXY_TYPE_HTTP,
"http_no_tunnel": socks.PROXY_TYPE_HTTP_NO_TUNNEL,
"socks4": socks.PROXY_TYPE_SOCKS4,
"socks5": socks.PROXY_TYPE_SOCKS5,
}
if config.get("proxy_type") in proxy_type_to_code:
proxy_type = proxy_type_to_code[config["proxy_type"]]
else:
proxy_type = socks.PROXY_TYPE_HTTP
rdns = scu.is_true(config.get("proxy_rdns"))
proxy_info = None
if config.get("proxy_url") and config.get("proxy_port"):
if config.get("proxy_username") and config.get("proxy_password"):
proxy_info = ProxyInfo(proxy_type=proxy_type,
proxy_host=config["proxy_url"],
proxy_port=int(config["proxy_port"]),
proxy_user=config["proxy_username"],
proxy_pass=config["proxy_password"],
proxy_rdns=rdns)
else:
proxy_info = ProxyInfo(proxy_type=proxy_type,
proxy_host=config["proxy_url"],
proxy_port=int(config["proxy_port"]),
proxy_rdns=rdns)
if proxy_info:
http = Http(proxy_info=proxy_info, timeout=timeout,
disable_ssl_certificate_validation=disable_ssl_validation)
else:
http = Http(timeout=timeout,
disable_ssl_certificate_validation=disable_ssl_validation)
if config.get("username") and config.get("password"):
http.add_credentials(config["username"], config["password"])
return http
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/schedule/__init__.py
View File
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/schedule/job.py
-90
View File
@@ -1,90 +0,0 @@
import threading
import time
class Job(object):
"""
Timer wraps the callback and timestamp related stuff
"""
_ident = 0
_lock = threading.Lock()
def __init__(self, func, job_props, interval, when=None, job_id=None):
"""
@job_props: dict like object
@func: execution function
@interval: execution interval
@when: seconds from epoch
@job_id: a unique id for the job
"""
self._props = job_props
self._func = func
if when is None:
self._when = time.time()
else:
self._when = when
self._interval = interval
if job_id is not None:
self._id = job_id
else:
with Job._lock:
self._id = Job._ident + 1
Job._ident = Job._ident + 1
self._stopped = False
def ident(self):
return self._id
def get_interval(self):
return self._interval
def set_interval(self, interval):
self._interval = interval
def get_expiration(self):
return self._when
def set_initial_due_time(self, when):
if self._when is None:
self._when = when
def update_expiration(self):
self._when += self._interval
def get(self, key, default):
return self._props.get(key, default)
def get_props(self):
return self._props
def set_props(self, props):
self._props = props
def __cmp__(self, other):
if other is None:
return 1
self_k = (self.get_expiration(), self.ident())
other_k = (other.get_expiration(), other.ident())
if self_k == other_k:
return 0
elif self_k < other_k:
return -1
else:
return 1
def __eq__(self, other):
return isinstance(other, Job) and (self.ident() == other.ident())
def __call__(self):
self._func(self)
def stop(self):
self._stopped = True
def stopped(self):
return self._stopped
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/schedule/scheduler.py
-143
View File
@@ -1,143 +0,0 @@
import threading
from time import time
import random
import Queue
from ..common import log
class Scheduler(object):
"""
A simple scheduler which schedules the periodic or once event
"""
import sortedcontainers as sc
max_delay_time = 60
def __init__(self):
self._jobs = Scheduler.sc.SortedSet()
self._wakeup_q = Queue.Queue()
self._lock = threading.Lock()
self._thr = threading.Thread(target=self._do_jobs)
self._thr.deamon = True
self._started = False
def start(self):
"""
Start the schduler which will start the internal thread for scheduling
jobs. Please do tear_down when doing cleanup
"""
if self._started:
log.logger.info("Scheduler already started.")
return
self._started = True
self._thr.start()
def tear_down(self):
"""
Stop the schduler which will stop the internal thread for scheduling
jobs.
"""
if not self._started:
log.logger.info("Scheduler already tear down.")
return
self._wakeup_q.put(True)
def _do_jobs(self):
while 1:
(sleep_time, jobs) = self.get_ready_jobs()
self._do_execution(jobs)
try:
done = self._wakeup_q.get(timeout=sleep_time)
except Queue.Empty:
pass
else:
if done:
break
self._started = False
log.logger.info("Scheduler exited.")
def get_ready_jobs(self):
"""
@return: a 2 element tuple. The first element is the next ready
duration. The second element is ready jobs list
"""
now = time()
ready_jobs = []
sleep_time = 1
with self._lock:
job_set = self._jobs
total_jobs = len(job_set)
for job in job_set:
if job.get_expiration() <= now:
ready_jobs.append(job)
if ready_jobs:
del job_set[:len(ready_jobs)]
for job in ready_jobs:
if job.get_interval() != 0 and not job.stopped():
# repeated job, calculate next due time and enqueue
job.update_expiration()
job_set.add(job)
if job_set:
sleep_time = job_set[0].get_expiration() - now
if sleep_time < 0:
log.logger.warn("Scheduler satuation, sleep_time=%s",
sleep_time)
sleep_time = 0.1
if ready_jobs:
log.logger.info("Get %d ready jobs, next duration is %f, "
"and there are %s jobs scheduling",
len(ready_jobs), sleep_time, total_jobs)
ready_jobs.sort(key=lambda job: job.get("priority", 0), reverse=True)
return (sleep_time, ready_jobs)
def add_jobs(self, jobs):
with self._lock:
now = time()
job_set = self._jobs
for job in jobs:
delay_time = random.randrange(0, self.max_delay_time)
job.set_initial_due_time(now + delay_time)
job_set.add(job)
self._wakeup()
def update_jobs(self, jobs):
with self._lock:
job_set = self._jobs
for njob in jobs:
job_set.discard(njob)
job_set.add(njob)
self._wakeup()
def remove_jobs(self, jobs):
with self._lock:
job_set = self._jobs
for njob in jobs:
njob.stop()
job_set.discard(njob)
self._wakeup()
def number_of_jobs(self):
with self._lock:
return len(self._jobs)
def disable_randomization(self):
self.max_delay_time = 1
def _wakeup(self):
self._wakeup_q.put(None)
def _do_execution(self, jobs):
for job in jobs:
job()
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/setting.conf
-5
View File
@@ -1,5 +0,0 @@
[global]
process_size = 0
thread_min_size = 4
thread_max_size = 128
task_queue_size = 1024
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/splunk_cluster.py
-53
View File
@@ -1,53 +0,0 @@
from ..splunktalib import rest
from ..splunktalib.common import xml_dom_parser as xdp
def _do_rest(uri, session_key):
resp, content = rest.splunkd_request(uri, session_key)
if resp is None:
return None
if resp.status not in (200, 201):
return None
stanza_objs = xdp.parse_conf_xml_dom(content)
if not stanza_objs:
return None
return stanza_objs[0]
class ServerInfo(object):
def __init__(self, splunkd_uri, session_key):
uri = "{}/services/server/info".format(splunkd_uri)
server_info = _do_rest(uri, session_key)
if server_info is None:
raise Exception("Failed to init ServerInfo")
self._server_info = server_info
def is_captain(self):
"""
:return: True if splunkd_uri is captain otherwise False
"""
return "shc_captain" in self._server_info["server_roles"]
def is_search_head(self):
for sh in ("search_head", "cluster_search_head"):
if sh in self._server_info["server_roles"]:
return True
return False
def is_shc_member(self):
server_roles = self._server_info['server_roles']
return any(
role in server_roles for role in ('shc_member', 'shc_captain')
)
def version(self):
return self._server_info["version"]
def to_dict(self):
return self._server_info
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/splunk_platform.py
-104
View File
@@ -1,104 +0,0 @@
import os
import os.path as op
import subprocess
from ConfigParser import ConfigParser
from cStringIO import StringIO
from .common import util as scu
def make_splunkhome_path(parts):
"""
create a path string by the several parts of the path
"""
relpath = os.path.normpath(os.path.join(*parts))
basepath = os.environ["SPLUNK_HOME"] # Assume SPLUNK_HOME env has been set
fullpath = os.path.normpath(os.path.join(basepath, relpath))
# Check that we haven't escaped from intended parent directories.
if os.path.relpath(fullpath, basepath)[0:2] == '..':
raise ValueError('Illegal escape from parent directory "%s": %s' %
(basepath, fullpath))
return fullpath
def get_splunk_bin():
if os.name == "nt":
splunk_bin = "splunk.exe"
else:
splunk_bin = "splunk"
return make_splunkhome_path(("bin", splunk_bin))
def get_appname_from_path(absolute_path):
return scu.get_appname_from_path(absolute_path)
def _get_merged_conf_raw(conf_name):
"""
:conf_name: configure file name
:return: raw output of all contents for the same conf file
Note: it depends on SPLUNK_HOME env variable
"""
assert conf_name
if conf_name.endswith(".conf"):
conf_name = conf_name[:-5]
# FIXME dynamically caculate SPLUNK_HOME
btool_cli = [op.join(os.environ["SPLUNK_HOME"], "bin", "btool"),
conf_name, "list"]
try:
p = subprocess.Popen(btool_cli, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
except OSError:
raise
return out
def _get_conf_stanzas(conf_name):
"""
:return: {stanza_name: stanza_configs}, dict of dict
"""
res = _get_merged_conf_raw(conf_name)
res = StringIO(res)
parser = ConfigParser()
parser.optionxform = str
parser.readfp(res)
res = {}
for section in parser.sections():
res[section] = {item[0]: item[1] for item in parser.items(section)}
return res
def get_splunkd_uri():
if "SPLUNKD_URI" in os.environ:
return os.environ["SPLUNKD_URI"]
else:
server_conf = _get_conf_stanzas("server")
if server_conf["sslConfig"]["enableSplunkdSSL"].lower() == "true":
http = "https://"
else:
http = "http://"
web_conf = _get_conf_stanzas("web")
host_port = web_conf["settings"]["mgmtHostPort"]
splunkd_uri = "{}{}".format(http, host_port)
if os.environ.get("SPLUNK_BINDIP"):
bip = os.environ["SPLUNK_BINDIP"]
port_idx = bip.rfind(":")
if port_idx > 0:
bip = bip[:port_idx]
port = host_port[host_port.rfind(":"):]
splunkd_uri = "{}{}{}".format(http, bip, port)
return splunkd_uri
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/state_store.py
-264
View File
@@ -1,264 +0,0 @@
import json
import os
import os.path as op
import time
from ..splunktacollectorlib.common import log as stulog
from ..splunktalib import kv_client as kvc
from ..splunktalib.common import util
def get_state_store(meta_configs,
appname,
collection_name="talib_states",
use_kv_store=False,
use_cache_file=True,
max_cache_seconds=5):
# FIXME refactor this
if util.is_true(use_kv_store):
return StateStore(meta_configs, appname, collection_name)
if util.is_true(use_cache_file):
return CachedFileStateStore(meta_configs, appname, max_cache_seconds)
return FileStateStore(meta_configs, appname)
class BaseStateStore(object):
def __init__(self, meta_configs, appname):
self._meta_configs = meta_configs
self._appname = appname
def update_state(self, key, states):
pass
def get_state(self, key):
pass
def delete_state(self, key):
pass
def close(self, key=None):
pass
class StateStore(BaseStateStore):
def __init__(self, meta_configs, appname, collection_name="talib_states"):
"""
:meta_configs: dict like and contains checkpoint_dir, session_key,
server_uri etc
:app_name: the name of the app
:collection_name: the collection name to be used.
Don"t use other method to visit the collection if you are using
StateStore to visit it.
"""
super(StateStore, self).__init__(meta_configs, appname)
# State cache is a dict from _key to value
self._states_cache = {}
self._kv_client = None
self._collection = collection_name
self._kv_client = kvc.KVClient(meta_configs["server_uri"],
meta_configs["session_key"])
kvc.create_collection(self._kv_client, self._collection, self._appname)
self._load_states_cache()
def update_state(self, key, states):
"""
:state: Any JSON serializable
:return: None if successful, otherwise throws exception
"""
if key not in self._states_cache:
self._kv_client.insert_collection_data(
self._collection, {"_key": key, "value": json.dumps(states)},
self._appname)
else:
self._kv_client.update_collection_data(
self._collection, key, {"value": json.dumps(states)},
self._appname)
self._states_cache[key] = states
def get_state(self, key=None):
if key:
return self._states_cache.get(key, None)
return self._states_cache
def delete_state(self, key=None):
if key:
self._delete_state(key)
else:
[self._delete_state(_key) for _key in self._states_cache.keys()]
def _delete_state(self, key):
if key not in self._states_cache:
return
self._kv_client.delete_collection_data(
self._collection, key, self._appname)
del self._states_cache[key]
def _load_states_cache(self):
states = self._kv_client.get_collection_data(
self._collection, None, self._appname)
if not states:
return
for state in states:
if "value" in state:
value = state["value"]
else:
value = state
try:
value = json.loads(value)
except Exception:
pass
self._states_cache[state["_key"]] = value
def _create_checkpoint_dir_if_needed(checkpoint_dir):
if os.path.isdir(checkpoint_dir):
return
stulog.logger.info(
"Checkpoint dir '%s' doesn't exist, try to create it",
checkpoint_dir)
try:
os.mkdir(checkpoint_dir)
except OSError:
stulog.logger.exception(
"Failure creating checkpoint dir '%s'", checkpoint_dir
)
raise Exception(
"Unable to create checkpoint dir '{}'".format(checkpoint_dir)
)
class FileStateStore(BaseStateStore):
def __init__(self, meta_configs, appname):
"""
:meta_configs: dict like and contains checkpoint_dir, session_key,
server_uri etc
"""
super(FileStateStore, self).__init__(meta_configs, appname)
def update_state(self, key, states):
"""
:state: Any JSON serializable
:return: None if successful, otherwise throws exception
"""
checkpoint_dir = self._meta_configs["checkpoint_dir"]
_create_checkpoint_dir_if_needed(checkpoint_dir)
fname = op.join(checkpoint_dir, key)
with open(fname + ".new", "w") as jsonfile:
json.dump(states, jsonfile)
if op.exists(fname):
os.remove(fname)
os.rename(fname + ".new", fname)
# commented this to disable state cache for local file
# if key not in self._states_cache:
# self._states_cache[key] = {}
# self._states_cache[key] = states
def get_state(self, key):
fname = op.join(self._meta_configs["checkpoint_dir"], key)
if op.exists(fname):
with open(fname) as jsonfile:
state = json.load(jsonfile)
# commented this to disable state cache for local file
# self._states_cache[key] = state
return state
else:
return None
def delete_state(self, key):
fname = op.join(self._meta_configs["checkpoint_dir"], key)
if op.exists(fname):
os.remove(fname)
class CachedFileStateStore(BaseStateStore):
def __init__(self, meta_configs, appname, max_cache_seconds=5):
"""
:meta_configs: dict like and contains checkpoint_dir, session_key,
server_uri etc
"""
super(CachedFileStateStore, self).__init__(meta_configs, appname)
self._states_cache = {} # item: time, dict
self._states_cache_lmd = {} #item: time, dict
self.max_cache_seconds = max_cache_seconds
def update_state(self, key, states):
now = time.time()
if key in self._states_cache:
last = self._states_cache_lmd[key][0]
if now - last >= self.max_cache_seconds:
self.update_state_flush(now, key, states)
else:
self.update_state_flush(now, key, states)
self._states_cache[key] = (now, states)
def update_state_flush(self, now, key, states):
"""
:state: Any JSON serializable
:return: None if successful, otherwise throws exception
"""
self._states_cache_lmd[key] = (now, states)
checkpoint_dir = self._meta_configs["checkpoint_dir"]
_create_checkpoint_dir_if_needed(checkpoint_dir)
fname = op.join(checkpoint_dir, key)
with open(fname + ".new", "w") as jsonfile:
json.dump(states, jsonfile)
if op.exists(fname):
os.remove(fname)
os.rename(fname + ".new", fname)
def get_state(self, key):
if key in self._states_cache:
return self._states_cache[key][1]
fname = op.join(self._meta_configs["checkpoint_dir"], key)
if op.exists(fname):
with open(fname) as jsonfile:
state = json.load(jsonfile)
now = time.time()
self._states_cache[key] = now, state
self._states_cache_lmd[key] = now, state
return state
else:
return None
def delete_state(self, key):
fname = op.join(self._meta_configs["checkpoint_dir"], key)
if op.exists(fname):
os.remove(fname)
if self._states_cache.get(key):
del self._states_cache[key]
if self._states_cache_lmd.get(key):
del self._states_cache_lmd[key]
def close(self, key=None):
if not key:
for k, (t, s) in self._states_cache.iteritems():
self.update_state_flush(t, k, s)
self._states_cache.clear()
self._states_cache_lmd.clear()
elif key in self._states_cache:
self.update_state_flush(self._states_cache[key][0], key,
self._states_cache[key][1])
del self._states_cache[key]
del self._states_cache_lmd[key]
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/timer.py
-60
View File
@@ -1,60 +0,0 @@
import threading
class Timer(object):
"""
Timer wraps the callback and timestamp related stuff
"""
_ident = 0
_lock = threading.Lock()
def __init__(self, callback, when, interval, ident=None):
self._callback = callback
self._when = when
self._interval = interval
if ident is not None:
self._id = ident
else:
with Timer._lock:
self._id = Timer._ident + 1
Timer._ident = Timer._ident + 1
def get_interval(self):
return self._interval
def set_interval(self, interval):
self._interval = interval
def get_expiration(self):
return self._when
def set_initial_due_time(self, when):
self._when = when
def update_expiration(self):
self._when += self._interval
def __cmp__(self, other):
if other is None:
return 1
self_k = (self.get_expiration(), self.ident())
other_k = (other.get_expiration(), other.ident())
if self_k == other_k:
return 0
elif self_k < other_k:
return -1
else:
return 1
def __eq__(self, other):
return isinstance(other, Timer) and (self.ident() == other.ident())
def __call__(self):
self._callback()
def ident(self):
return self._id
package/bin/da_ess_contentupdate/cloudconnectlib/splunktalib/timer_queue.py
-138
View File
@@ -1,138 +0,0 @@
"""
A timer queue implementation
"""
import threading
import Queue
from time import time
import traceback
from .timer import Timer
from .common import log
class TimerQueue(object):
"""
A timer queue implementation, runs a separate thread to handle timers
"""
import sortedcontainers as sc
def __init__(self):
self._timers = TimerQueue.sc.SortedSet()
self._cancelling_timers = {}
self._lock = threading.Lock()
self._wakeup_queue = Queue.Queue()
self._thr = threading.Thread(target=self._check_and_execute)
self._started = False
def start(self):
"""
Start the timer queue to make it start function
"""
if self._started:
return
self._started = True
self._thr.start()
log.logger.info("TimerQueue started.")
def tear_down(self):
if not self._started:
return
self._started = True
self._wakeup(None)
self._thr.join()
def add_timer(self, callback, when, interval):
"""
Add timer to the queue
"""
timer = Timer(callback, when, interval)
with self._lock:
self._timers.add(timer)
self._wakeup()
return timer
def remove_timer(self, timer):
"""
Remove timer from the queue.
"""
with self._lock:
try:
self._timers.remove(timer)
except ValueError:
log.logger.info("Timer=%s is not in queue, move it to cancelling "
"list", timer.ident())
else:
self._cancelling_timers[timer.ident()] = timer
def _check_and_execute(self):
wakeup_queue = self._wakeup_queue
while 1:
(next_expired_time, expired_timers) = self._get_expired_timers()
for timer in expired_timers:
try:
timer()
except Exception:
log.logger.error(traceback.format_exc())
self._reset_timers(expired_timers)
# Calc sleep time
if next_expired_time:
now = time()
if now < next_expired_time:
sleep_time = next_expired_time - now
else:
sleep_time = 0.1
else:
sleep_time = 1
try:
wakeup = wakeup_queue.get(timeout=sleep_time)
if wakeup is None:
break
except Queue.Empty:
pass
log.logger.info("TimerQueue stopped.")
def _get_expired_timers(self):
next_expired_time = 0
now = time()
expired_timers = []
with self._lock:
for timer in self._timers:
if timer.get_expiration() <= now:
expired_timers.append(timer)
if expired_timers:
del self._timers[:len(expired_timers)]
if self._timers:
next_expired_time = self._timers[0].get_expiration()
return (next_expired_time, expired_timers)
def _reset_timers(self, expired_timers):
has_new_timer = False
with self._lock:
cancelling_timers = self._cancelling_timers
for timer in expired_timers:
if timer.ident() in cancelling_timers:
log.logger.INFO("Timer=%s has been cancelled", timer.ident())
continue
elif timer.get_interval():
# Repeated timer
timer.update_expiration()
self._timers.add(timer)
has_new_timer = True
cancelling_timers.clear()
if has_new_timer:
self._wakeup()
def _wakeup(self, something="not_None"):
self._wakeup_queue.put(something)
package/bin/da_ess_contentupdate/decorator.py
-414
View File
@@ -1,414 +0,0 @@
# ######################### LICENSE ############################ #
# Copyright (c) 2005-2017, Michele Simionato
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# Redistributions in bytecode form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.
"""
Decorator module, see http://pypi.python.org/pypi/decorator
for the documentation.
"""
from __future__ import print_function
import re
import sys
import inspect
import operator
import itertools
import collections
__version__ = '4.0.11'
if sys.version >= '3':
from inspect import getfullargspec
def get_init(cls):
return cls.__init__
else:
FullArgSpec = collections.namedtuple(
'FullArgSpec', 'args varargs varkw defaults '
'kwonlyargs kwonlydefaults')
def getfullargspec(f):
"A quick and dirty replacement for getfullargspec for Python 2.X"
return FullArgSpec._make(inspect.getargspec(f) + ([], None))
def get_init(cls):
return cls.__init__.__func__
# getargspec has been deprecated in Python 3.5
ArgSpec = collections.namedtuple(
'ArgSpec', 'args varargs varkw defaults')
def getargspec(f):
"""A replacement for inspect.getargspec"""
spec = getfullargspec(f)
return ArgSpec(spec.args, spec.varargs, spec.varkw, spec.defaults)
DEF = re.compile(r'\s*def\s*([_\w][_\w\d]*)\s*\(')
# basic functionality
class FunctionMaker(object):
"""
An object with the ability to create functions with a given signature.
It has attributes name, doc, module, signature, defaults, dict and
methods update and make.
"""
# Atomic get-and-increment provided by the GIL
_compile_count = itertools.count()
# make pylint happy
args = varargs = varkw = defaults = kwonlyargs = kwonlydefaults = ()
def __init__(self, func=None, name=None, signature=None,
defaults=None, doc=None, module=None, funcdict=None):
self.shortsignature = signature
if func:
# func can be a class or a callable, but not an instance method
self.name = func.__name__
if self.name == '<lambda>': # small hack for lambda functions
self.name = '_lambda_'
self.doc = func.__doc__
self.module = func.__module__
if inspect.isfunction(func):
argspec = getfullargspec(func)
self.annotations = getattr(func, '__annotations__', {})
for a in ('args', 'varargs', 'varkw', 'defaults', 'kwonlyargs',
'kwonlydefaults'):
setattr(self, a, getattr(argspec, a))
for i, arg in enumerate(self.args):
setattr(self, 'arg%d' % i, arg)
if sys.version < '3': # easy way
self.shortsignature = self.signature = (
inspect.formatargspec(
formatvalue=lambda val: "", *argspec[:-2])[1:-1])
else: # Python 3 way
allargs = list(self.args)
allshortargs = list(self.args)
if self.varargs:
allargs.append('*' + self.varargs)
allshortargs.append('*' + self.varargs)
elif self.kwonlyargs:
allargs.append('*') # single star syntax
for a in self.kwonlyargs:
allargs.append('%s=None' % a)
allshortargs.append('%s=%s' % (a, a))
if self.varkw:
allargs.append('**' + self.varkw)
allshortargs.append('**' + self.varkw)
self.signature = ', '.join(allargs)
self.shortsignature = ', '.join(allshortargs)
self.dict = func.__dict__.copy()
# func=None happens when decorating a caller
if name:
self.name = name
if signature is not None:
self.signature = signature
if defaults:
self.defaults = defaults
if doc:
self.doc = doc
if module:
self.module = module
if funcdict:
self.dict = funcdict
# check existence required attributes
assert hasattr(self, 'name')
if not hasattr(self, 'signature'):
raise TypeError('You are decorating a non function: %s' % func)
def update(self, func, **kw):
"Update the signature of func with the data in self"
func.__name__ = self.name
func.__doc__ = getattr(self, 'doc', None)
func.__dict__ = getattr(self, 'dict', {})
func.__defaults__ = self.defaults
func.__kwdefaults__ = self.kwonlydefaults or None
func.__annotations__ = getattr(self, 'annotations', None)
try:
frame = sys._getframe(3)
except AttributeError: # for IronPython and similar implementations
callermodule = '?'
else:
callermodule = frame.f_globals.get('__name__', '?')
func.__module__ = getattr(self, 'module', callermodule)
func.__dict__.update(kw)
def make(self, src_templ, evaldict=None, addsource=False, **attrs):
"Make a new function from a given template and update the signature"
src = src_templ % vars(self) # expand name and signature
evaldict = evaldict or {}
mo = DEF.match(src)
if mo is None:
raise SyntaxError('not a valid function template\n%s' % src)
name = mo.group(1) # extract the function name
names = set([name] + [arg.strip(' *') for arg in
self.shortsignature.split(',')])
for n in names:
if n in ('_func_', '_call_'):
raise NameError('%s is overridden in\n%s' % (n, src))
if not src.endswith('\n'): # add a newline for old Pythons
src += '\n'
# Ensure each generated function has a unique filename for profilers
# (such as cProfile) that depend on the tuple of (<filename>,
# <definition line>, <function name>) being unique.
filename = '<decorator-gen-%d>' % (next(self._compile_count),)
try:
code = compile(src, filename, 'single')
exec(code, evaldict)
except:
print('Error in generated code:', file=sys.stderr)
print(src, file=sys.stderr)
raise
func = evaldict[name]
if addsource:
attrs['__source__'] = src
self.update(func, **attrs)
return func
@classmethod
def create(cls, obj, body, evaldict, defaults=None,
doc=None, module=None, addsource=True, **attrs):
"""
Create a function from the strings name, signature and body.
evaldict is the evaluation dictionary. If addsource is true an
attribute __source__ is added to the result. The attributes attrs
are added, if any.
"""
if isinstance(obj, str): # "name(signature)"
name, rest = obj.strip().split('(', 1)
signature = rest[:-1] # strip a right parens
func = None
else: # a function
name = None
signature = None
func = obj
self = cls(func, name, signature, defaults, doc, module)
ibody = '\n'.join(' ' + line for line in body.splitlines())
return self.make('def %(name)s(%(signature)s):\n' + ibody,
evaldict, addsource, **attrs)
def decorate(func, caller):
"""
decorate(func, caller) decorates a function using a caller.
"""
evaldict = dict(_call_=caller, _func_=func)
fun = FunctionMaker.create(
func, "return _call_(_func_, %(shortsignature)s)",
evaldict, __wrapped__=func)
if hasattr(func, '__qualname__'):
fun.__qualname__ = func.__qualname__
return fun
def decorator(caller, _func=None):
"""decorator(caller) converts a caller function into a decorator"""
if _func is not None: # return a decorated function
# this is obsolete behavior; you should use decorate instead
return decorate(_func, caller)
# else return a decorator function
if inspect.isclass(caller):
name = caller.__name__.lower()
doc = 'decorator(%s) converts functions/generators into ' \
'factories of %s objects' % (caller.__name__, caller.__name__)
elif inspect.isfunction(caller):
if caller.__name__ == '<lambda>':
name = '_lambda_'
else:
name = caller.__name__
doc = caller.__doc__
else: # assume caller is an object with a __call__ method
name = caller.__class__.__name__.lower()
doc = caller.__call__.__doc__
evaldict = dict(_call_=caller, _decorate_=decorate)
return FunctionMaker.create(
'%s(func)' % name, 'return _decorate_(func, _call_)',
evaldict, doc=doc, module=caller.__module__,
__wrapped__=caller)
# ####################### contextmanager ####################### #
try: # Python >= 3.2
from contextlib import _GeneratorContextManager
except ImportError: # Python >= 2.5
from contextlib import GeneratorContextManager as _GeneratorContextManager
class ContextManager(_GeneratorContextManager):
def __call__(self, func):
"""Context manager decorator"""
return FunctionMaker.create(
func, "with _self_: return _func_(%(shortsignature)s)",
dict(_self_=self, _func_=func), __wrapped__=func)
init = getfullargspec(_GeneratorContextManager.__init__)
n_args = len(init.args)
if n_args == 2 and not init.varargs: # (self, genobj) Python 2.7
def __init__(self, g, *a, **k):
return _GeneratorContextManager.__init__(self, g(*a, **k))
ContextManager.__init__ = __init__
elif n_args == 2 and init.varargs: # (self, gen, *a, **k) Python 3.4
pass
elif n_args == 4: # (self, gen, args, kwds) Python 3.5
def __init__(self, g, *a, **k):
return _GeneratorContextManager.__init__(self, g, a, k)
ContextManager.__init__ = __init__
contextmanager = decorator(ContextManager)
# ############################ dispatch_on ############################ #
def append(a, vancestors):
"""
Append ``a`` to the list of the virtual ancestors, unless it is already
included.
"""
add = True
for j, va in enumerate(vancestors):
if issubclass(va, a):
add = False
break
if issubclass(a, va):
vancestors[j] = a
add = False
if add:
vancestors.append(a)
# inspired from simplegeneric by P.J. Eby and functools.singledispatch
def dispatch_on(*dispatch_args):
"""
Factory of decorators turning a function into a generic function
dispatching on the given arguments.
"""
assert dispatch_args, 'No dispatch args passed'
dispatch_str = '(%s,)' % ', '.join(dispatch_args)
def check(arguments, wrong=operator.ne, msg=''):
"""Make sure one passes the expected number of arguments"""
if wrong(len(arguments), len(dispatch_args)):
raise TypeError('Expected %d arguments, got %d%s' %
(len(dispatch_args), len(arguments), msg))
def gen_func_dec(func):
"""Decorator turning a function into a generic function"""
# first check the dispatch arguments
argset = set(getfullargspec(func).args)
if not set(dispatch_args) <= argset:
raise NameError('Unknown dispatch arguments %s' % dispatch_str)
typemap = {}
def vancestors(*types):
"""
Get a list of sets of virtual ancestors for the given types
"""
check(types)
ras = [[] for _ in range(len(dispatch_args))]
for types_ in typemap:
for t, type_, ra in zip(types, types_, ras):
if issubclass(t, type_) and type_ not in t.mro():
append(type_, ra)
return [set(ra) for ra in ras]
def ancestors(*types):
"""
Get a list of virtual MROs, one for each type
"""
check(types)
lists = []
for t, vas in zip(types, vancestors(*types)):
n_vas = len(vas)
if n_vas > 1:
raise RuntimeError(
'Ambiguous dispatch for %s: %s' % (t, vas))
elif n_vas == 1:
va, = vas
mro = type('t', (t, va), {}).mro()[1:]
else:
mro = t.mro()
lists.append(mro[:-1]) # discard t and object
return lists
def register(*types):
"""
Decorator to register an implementation for the given types
"""
check(types)
def dec(f):
check(getfullargspec(f).args, operator.lt, ' in ' + f.__name__)
typemap[types] = f
return f
return dec
def dispatch_info(*types):
"""
An utility to introspect the dispatch algorithm
"""
check(types)
lst = []
for anc in itertools.product(*ancestors(*types)):
lst.append(tuple(a.__name__ for a in anc))
return lst
def _dispatch(dispatch_args, *args, **kw):
types = tuple(type(arg) for arg in dispatch_args)
try: # fast path
f = typemap[types]
except KeyError:
pass
else:
return f(*args, **kw)
combinations = itertools.product(*ancestors(*types))
next(combinations) # the first one has been already tried
for types_ in combinations:
f = typemap.get(types_)
if f is not None:
return f(*args, **kw)
# else call the default implementation
return func(*args, **kw)
return FunctionMaker.create(
func, 'return _f_(%s, %%(shortsignature)s)' % dispatch_str,
dict(_f_=_dispatch), register=register, default=func,
typemap=typemap, vancestors=vancestors, ancestors=ancestors,
dispatch_info=dispatch_info, __wrapped__=func)
gen_func_dec.__name__ = 'dispatch_on' + dispatch_str
return gen_func_dec
package/bin/da_ess_contentupdate/functools32/__init__.py
-1
View File
@@ -1 +0,0 @@
from .functools32 import *
package/bin/da_ess_contentupdate/functools32/_dummy_thread32.py
-158
View File
@@ -1,158 +0,0 @@
"""Drop-in replacement for the thread module.
Meant to be used as a brain-dead substitute so that threaded code does
not need to be rewritten for when the thread module is not present.
Suggested usage is::
try:
try:
import _thread # Python >= 3
except:
import thread as _thread # Python < 3
except ImportError:
import _dummy_thread as _thread
"""
# Exports only things specified by thread documentation;
# skipping obsolete synonyms allocate(), start_new(), exit_thread().
__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
'interrupt_main', 'LockType']
# A dummy value
TIMEOUT_MAX = 2**31
# NOTE: this module can be imported early in the extension building process,
# and so top level imports of other modules should be avoided. Instead, all
# imports are done when needed on a function-by-function basis. Since threads
# are disabled, the import lock should not be an issue anyway (??).
class error(Exception):
"""Dummy implementation of _thread.error."""
def __init__(self, *args):
self.args = args
def start_new_thread(function, args, kwargs={}):
"""Dummy implementation of _thread.start_new_thread().
Compatibility is maintained by making sure that ``args`` is a
tuple and ``kwargs`` is a dictionary. If an exception is raised
and it is SystemExit (which can be done by _thread.exit()) it is
caught and nothing is done; all other exceptions are printed out
by using traceback.print_exc().
If the executed function calls interrupt_main the KeyboardInterrupt will be
raised when the function returns.
"""
if type(args) != type(tuple()):
raise TypeError("2nd arg must be a tuple")
if type(kwargs) != type(dict()):
raise TypeError("3rd arg must be a dict")
global _main
_main = False
try:
function(*args, **kwargs)
except SystemExit:
pass
except:
import traceback
traceback.print_exc()
_main = True
global _interrupt
if _interrupt:
_interrupt = False
raise KeyboardInterrupt
def exit():
"""Dummy implementation of _thread.exit()."""
raise SystemExit
def get_ident():
"""Dummy implementation of _thread.get_ident().
Since this module should only be used when _threadmodule is not
available, it is safe to assume that the current process is the
only thread. Thus a constant can be safely returned.
"""
return -1
def allocate_lock():
"""Dummy implementation of _thread.allocate_lock()."""
return LockType()
def stack_size(size=None):
"""Dummy implementation of _thread.stack_size()."""
if size is not None:
raise error("setting thread stack size not supported")
return 0
class LockType(object):
"""Class implementing dummy implementation of _thread.LockType.
Compatibility is maintained by maintaining self.locked_status
which is a boolean that stores the state of the lock. Pickling of
the lock, though, should not be done since if the _thread module is
then used with an unpickled ``lock()`` from here problems could
occur from this class not having atomic methods.
"""
def __init__(self):
self.locked_status = False
def acquire(self, waitflag=None, timeout=-1):
"""Dummy implementation of acquire().
For blocking calls, self.locked_status is automatically set to
True and returned appropriately based on value of
``waitflag``. If it is non-blocking, then the value is
actually checked and not set if it is already acquired. This
is all done so that threading.Condition's assert statements
aren't triggered and throw a little fit.
"""
if waitflag is None or waitflag:
self.locked_status = True
return True
else:
if not self.locked_status:
self.locked_status = True
return True
else:
if timeout > 0:
import time
time.sleep(timeout)
return False
__enter__ = acquire
def __exit__(self, typ, val, tb):
self.release()
def release(self):
"""Release the dummy lock."""
# XXX Perhaps shouldn't actually bother to test? Could lead
# to problems for complex, threaded code.
if not self.locked_status:
raise error
self.locked_status = False
return True
def locked(self):
return self.locked_status
# Used to signal that interrupt_main was called in a "thread"
_interrupt = False
# True when not executing in a "thread"
_main = True
def interrupt_main():
"""Set _interrupt flag to True to have start_new_thread raise
KeyboardInterrupt upon exiting."""
if _main:
raise KeyboardInterrupt
else:
global _interrupt
_interrupt = True
package/bin/da_ess_contentupdate/functools32/functools32.py
-423
View File
@@ -1,423 +0,0 @@
"""functools.py - Tools for working with functions and callable objects
"""
# Python module wrapper for _functools C module
# to allow utilities written in Python to be added
# to the functools module.
# Written by Nick Coghlan <ncoghlan at gmail.com>
# and Raymond Hettinger <python at rcn.com>
# Copyright (C) 2006-2010 Python Software Foundation.
# See C source code for _functools credits/copyright
__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial']
from _functools import partial, reduce
from collections import MutableMapping, namedtuple
from .reprlib32 import recursive_repr as _recursive_repr
from weakref import proxy as _proxy
import sys as _sys
try:
from thread import allocate_lock as Lock
except ImportError:
from ._dummy_thread32 import allocate_lock as Lock
################################################################################
### OrderedDict
################################################################################
class _Link(object):
__slots__ = 'prev', 'next', 'key', '__weakref__'
class OrderedDict(dict):
'Dictionary that remembers insertion order'
# An inherited dict maps keys to values.
# The inherited dict provides __getitem__, __len__, __contains__, and get.
# The remaining methods are order-aware.
# Big-O running times for all methods are the same as regular dictionaries.
# The internal self.__map dict maps keys to links in a doubly linked list.
# The circular doubly linked list starts and ends with a sentinel element.
# The sentinel element never gets deleted (this simplifies the algorithm).
# The sentinel is in self.__hardroot with a weakref proxy in self.__root.
# The prev links are weakref proxies (to prevent circular references).
# Individual links are kept alive by the hard reference in self.__map.
# Those hard references disappear when a key is deleted from an OrderedDict.
def __init__(self, *args, **kwds):
'''Initialize an ordered dictionary. The signature is the same as
regular dictionaries, but keyword arguments are not recommended because
their insertion order is arbitrary.
'''
if len(args) > 1:
raise TypeError('expected at most 1 arguments, got %d' % len(args))
try:
self.__root
except AttributeError:
self.__hardroot = _Link()
self.__root = root = _proxy(self.__hardroot)
root.prev = root.next = root
self.__map = {}
self.__update(*args, **kwds)
def __setitem__(self, key, value,
dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link):
'od.__setitem__(i, y) <==> od[i]=y'
# Setting a new item creates a new link at the end of the linked list,
# and the inherited dictionary is updated with the new key/value pair.
if key not in self:
self.__map[key] = link = Link()
root = self.__root
last = root.prev
link.prev, link.next, link.key = last, root, key
last.next = link
root.prev = proxy(link)
dict_setitem(self, key, value)
def __delitem__(self, key, dict_delitem=dict.__delitem__):
'od.__delitem__(y) <==> del od[y]'
# Deleting an existing item uses self.__map to find the link which gets
# removed by updating the links in the predecessor and successor nodes.
dict_delitem(self, key)
link = self.__map.pop(key)
link_prev = link.prev
link_next = link.next
link_prev.next = link_next
link_next.prev = link_prev
def __iter__(self):
'od.__iter__() <==> iter(od)'
# Traverse the linked list in order.
root = self.__root
curr = root.next
while curr is not root:
yield curr.key
curr = curr.next
def __reversed__(self):
'od.__reversed__() <==> reversed(od)'
# Traverse the linked list in reverse order.
root = self.__root
curr = root.prev
while curr is not root:
yield curr.key
curr = curr.prev
def clear(self):
'od.clear() -> None. Remove all items from od.'
root = self.__root
root.prev = root.next = root
self.__map.clear()
dict.clear(self)
def popitem(self, last=True):
'''od.popitem() -> (k, v), return and remove a (key, value) pair.
Pairs are returned in LIFO order if last is true or FIFO order if false.
'''
if not self:
raise KeyError('dictionary is empty')
root = self.__root
if last:
link = root.prev
link_prev = link.prev
link_prev.next = root
root.prev = link_prev
else:
link = root.next
link_next = link.next
root.next = link_next
link_next.prev = root
key = link.key
del self.__map[key]
value = dict.pop(self, key)
return key, value
def move_to_end(self, key, last=True):
'''Move an existing element to the end (or beginning if last==False).
Raises KeyError if the element does not exist.
When last=True, acts like a fast version of self[key]=self.pop(key).
'''
link = self.__map[key]
link_prev = link.prev
link_next = link.next
link_prev.next = link_next
link_next.prev = link_prev
root = self.__root
if last:
last = root.prev
link.prev = last
link.next = root
last.next = root.prev = link
else:
first = root.next
link.prev = root
link.next = first
root.next = first.prev = link
def __sizeof__(self):
sizeof = _sys.getsizeof
n = len(self) + 1 # number of links including root
size = sizeof(self.__dict__) # instance dictionary
size += sizeof(self.__map) * 2 # internal dict and inherited dict
size += sizeof(self.__hardroot) * n # link objects
size += sizeof(self.__root) * n # proxy objects
return size
update = __update = MutableMapping.update
keys = MutableMapping.keys
values = MutableMapping.values
items = MutableMapping.items
__ne__ = MutableMapping.__ne__
__marker = object()
def pop(self, key, default=__marker):
'''od.pop(k[,d]) -> v, remove specified key and return the corresponding
value. If key is not found, d is returned if given, otherwise KeyError
is raised.
'''
if key in self:
result = self[key]
del self[key]
return result
if default is self.__marker:
raise KeyError(key)
return default
def setdefault(self, key, default=None):
'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
if key in self:
return self[key]
self[key] = default
return default
@_recursive_repr()
def __repr__(self):
'od.__repr__() <==> repr(od)'
if not self:
return '%s()' % (self.__class__.__name__,)
return '%s(%r)' % (self.__class__.__name__, list(self.items()))
def __reduce__(self):
'Return state information for pickling'
items = [[k, self[k]] for k in self]
inst_dict = vars(self).copy()
for k in vars(OrderedDict()):
inst_dict.pop(k, None)
if inst_dict:
return (self.__class__, (items,), inst_dict)
return self.__class__, (items,)
def copy(self):
'od.copy() -> a shallow copy of od'
return self.__class__(self)
@classmethod
def fromkeys(cls, iterable, value=None):
'''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
If not specified, the value defaults to None.
'''
self = cls()
for key in iterable:
self[key] = value
return self
def __eq__(self, other):
'''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
while comparison to a regular mapping is order-insensitive.
'''
if isinstance(other, OrderedDict):
return len(self)==len(other) and \
all(p==q for p, q in zip(self.items(), other.items()))
return dict.__eq__(self, other)
# update_wrapper() and wraps() are tools to help write
# wrapper functions that can handle naive introspection
WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__')
WRAPPER_UPDATES = ('__dict__',)
def update_wrapper(wrapper,
wrapped,
assigned = WRAPPER_ASSIGNMENTS,
updated = WRAPPER_UPDATES):
"""Update a wrapper function to look like the wrapped function
wrapper is the function to be updated
wrapped is the original function
assigned is a tuple naming the attributes assigned directly
from the wrapped function to the wrapper function (defaults to
functools.WRAPPER_ASSIGNMENTS)
updated is a tuple naming the attributes of the wrapper that
are updated with the corresponding attribute from the wrapped
function (defaults to functools.WRAPPER_UPDATES)
"""
wrapper.__wrapped__ = wrapped
for attr in assigned:
try:
value = getattr(wrapped, attr)
except AttributeError:
pass
else:
setattr(wrapper, attr, value)
for attr in updated:
getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
# Return the wrapper so this can be used as a decorator via partial()
return wrapper
def wraps(wrapped,
assigned = WRAPPER_ASSIGNMENTS,
updated = WRAPPER_UPDATES):
"""Decorator factory to apply update_wrapper() to a wrapper function
Returns a decorator that invokes update_wrapper() with the decorated
function as the wrapper argument and the arguments to wraps() as the
remaining arguments. Default arguments are as for update_wrapper().
This is a convenience function to simplify applying partial() to
update_wrapper().
"""
return partial(update_wrapper, wrapped=wrapped,
assigned=assigned, updated=updated)
def total_ordering(cls):
"""Class decorator that fills in missing ordering methods"""
convert = {
'__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
('__le__', lambda self, other: self < other or self == other),
('__ge__', lambda self, other: not self < other)],
'__le__': [('__ge__', lambda self, other: not self <= other or self == other),
('__lt__', lambda self, other: self <= other and not self == other),
('__gt__', lambda self, other: not self <= other)],
'__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
('__ge__', lambda self, other: self > other or self == other),
('__le__', lambda self, other: not self > other)],
'__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
('__gt__', lambda self, other: self >= other and not self == other),
('__lt__', lambda self, other: not self >= other)]
}
roots = set(dir(cls)) & set(convert)
if not roots:
raise ValueError('must define at least one ordering operation: < > <= >=')
root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
for opname, opfunc in convert[root]:
if opname not in roots:
opfunc.__name__ = opname
opfunc.__doc__ = getattr(int, opname).__doc__
setattr(cls, opname, opfunc)
return cls
def cmp_to_key(mycmp):
"""Convert a cmp= function into a key= function"""
class K(object):
__slots__ = ['obj']
def __init__(self, obj):
self.obj = obj
def __lt__(self, other):
return mycmp(self.obj, other.obj) < 0
def __gt__(self, other):
return mycmp(self.obj, other.obj) > 0
def __eq__(self, other):
return mycmp(self.obj, other.obj) == 0
def __le__(self, other):
return mycmp(self.obj, other.obj) <= 0
def __ge__(self, other):
return mycmp(self.obj, other.obj) >= 0
def __ne__(self, other):
return mycmp(self.obj, other.obj) != 0
__hash__ = None
return K
_CacheInfo = namedtuple("CacheInfo", "hits misses maxsize currsize")
def lru_cache(maxsize=100):
"""Least-recently-used cache decorator.
If *maxsize* is set to None, the LRU features are disabled and the cache
can grow without bound.
Arguments to the cached function must be hashable.
View the cache statistics named tuple (hits, misses, maxsize, currsize) with
f.cache_info(). Clear the cache and statistics with f.cache_clear().
Access the underlying function with f.__wrapped__.
See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
"""
# Users should only access the lru_cache through its public API:
# cache_info, cache_clear, and f.__wrapped__
# The internals of the lru_cache are encapsulated for thread safety and
# to allow the implementation to change (including a possible C version).
def decorating_function(user_function,
tuple=tuple, sorted=sorted, len=len, KeyError=KeyError):
hits, misses = [0], [0]
kwd_mark = (object(),) # separates positional and keyword args
lock = Lock() # needed because OrderedDict isn't threadsafe
if maxsize is None:
cache = dict() # simple cache without ordering or size limit
@wraps(user_function)
def wrapper(*args, **kwds):
key = args
if kwds:
key += kwd_mark + tuple(sorted(kwds.items()))
try:
result = cache[key]
hits[0] += 1
return result
except KeyError:
pass
result = user_function(*args, **kwds)
cache[key] = result
misses[0] += 1
return result
else:
cache = OrderedDict() # ordered least recent to most recent
cache_popitem = cache.popitem
cache_renew = cache.move_to_end
@wraps(user_function)
def wrapper(*args, **kwds):
key = args
if kwds:
key += kwd_mark + tuple(sorted(kwds.items()))
with lock:
try:
result = cache[key]
cache_renew(key) # record recent use of this key
hits[0] += 1
return result
except KeyError:
pass
result = user_function(*args, **kwds)
with lock:
cache[key] = result # record recent use of this key
misses[0] += 1
if len(cache) > maxsize:
cache_popitem(0) # purge least recently used cache entry
return result
def cache_info():
"""Report cache statistics"""
with lock:
return _CacheInfo(hits[0], misses[0], maxsize, len(cache))
def cache_clear():
"""Clear the cache and cache statistics"""
with lock:
cache.clear()
hits[0] = misses[0] = 0
wrapper.cache_info = cache_info
wrapper.cache_clear = cache_clear
return wrapper
return decorating_function
package/bin/da_ess_contentupdate/functools32/reprlib32.py
-157
View File
@@ -1,157 +0,0 @@
"""Redo the builtin repr() (representation) but with limits on most sizes."""
__all__ = ["Repr", "repr", "recursive_repr"]
import __builtin__ as builtins
from itertools import islice
try:
from thread import get_ident
except ImportError:
from _dummy_thread32 import get_ident
def recursive_repr(fillvalue='...'):
'Decorator to make a repr function return fillvalue for a recursive call'
def decorating_function(user_function):
repr_running = set()
def wrapper(self):
key = id(self), get_ident()
if key in repr_running:
return fillvalue
repr_running.add(key)
try:
result = user_function(self)
finally:
repr_running.discard(key)
return result
# Can't use functools.wraps() here because of bootstrap issues
wrapper.__module__ = getattr(user_function, '__module__')
wrapper.__doc__ = getattr(user_function, '__doc__')
wrapper.__name__ = getattr(user_function, '__name__')
wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
return wrapper
return decorating_function
class Repr:
def __init__(self):
self.maxlevel = 6
self.maxtuple = 6
self.maxlist = 6
self.maxarray = 5
self.maxdict = 4
self.maxset = 6
self.maxfrozenset = 6
self.maxdeque = 6
self.maxstring = 30
self.maxlong = 40
self.maxother = 30
def repr(self, x):
return self.repr1(x, self.maxlevel)
def repr1(self, x, level):
typename = type(x).__name__
if ' ' in typename:
parts = typename.split()
typename = '_'.join(parts)
if hasattr(self, 'repr_' + typename):
return getattr(self, 'repr_' + typename)(x, level)
else:
return self.repr_instance(x, level)
def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
n = len(x)
if level <= 0 and n:
s = '...'
else:
newlevel = level - 1
repr1 = self.repr1
pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)]
if n > maxiter: pieces.append('...')
s = ', '.join(pieces)
if n == 1 and trail: right = trail + right
return '%s%s%s' % (left, s, right)
def repr_tuple(self, x, level):
return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',')
def repr_list(self, x, level):
return self._repr_iterable(x, level, '[', ']', self.maxlist)
def repr_array(self, x, level):
header = "array('%s', [" % x.typecode
return self._repr_iterable(x, level, header, '])', self.maxarray)
def repr_set(self, x, level):
x = _possibly_sorted(x)
return self._repr_iterable(x, level, 'set([', '])', self.maxset)
def repr_frozenset(self, x, level):
x = _possibly_sorted(x)
return self._repr_iterable(x, level, 'frozenset([', '])',
self.maxfrozenset)
def repr_deque(self, x, level):
return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque)
def repr_dict(self, x, level):
n = len(x)
if n == 0: return '{}'
if level <= 0: return '{...}'
newlevel = level - 1
repr1 = self.repr1
pieces = []
for key in islice(_possibly_sorted(x), self.maxdict):
keyrepr = repr1(key, newlevel)
valrepr = repr1(x[key], newlevel)
pieces.append('%s: %s' % (keyrepr, valrepr))
if n > self.maxdict: pieces.append('...')
s = ', '.join(pieces)
return '{%s}' % (s,)
def repr_str(self, x, level):
s = builtins.repr(x[:self.maxstring])
if len(s) > self.maxstring:
i = max(0, (self.maxstring-3)//2)
j = max(0, self.maxstring-3-i)
s = builtins.repr(x[:i] + x[len(x)-j:])
s = s[:i] + '...' + s[len(s)-j:]
return s
def repr_int(self, x, level):
s = builtins.repr(x) # XXX Hope this isn't too slow...
if len(s) > self.maxlong:
i = max(0, (self.maxlong-3)//2)
j = max(0, self.maxlong-3-i)
s = s[:i] + '...' + s[len(s)-j:]
return s
def repr_instance(self, x, level):
try:
s = builtins.repr(x)
# Bugs in x.__repr__() can cause arbitrary
# exceptions -- then make up something
except Exception:
return '<%s instance at %x>' % (x.__class__.__name__, id(x))
if len(s) > self.maxother:
i = max(0, (self.maxother-3)//2)
j = max(0, self.maxother-3-i)
s = s[:i] + '...' + s[len(s)-j:]
return s
def _possibly_sorted(x):
# Since not all sequences of items can be sorted and comparison
# functions may raise arbitrary exceptions, return an unsorted
# sequence in that case.
try:
return sorted(x)
except Exception:
return list(x)
aRepr = Repr()
repr = aRepr.repr
package/bin/da_ess_contentupdate/httplib2/__init__.py
-1696
View File
File diff suppressed because it is too large Load Diff
package/bin/da_ess_contentupdate/httplib2/cacerts.txt
-2183
View File
File diff suppressed because it is too large Load Diff
package/bin/da_ess_contentupdate/httplib2/iri2uri.py
-110
View File
@@ -1,110 +0,0 @@
"""
iri2uri
Converts an IRI to a URI.
"""
__author__ = "Joe Gregorio (joe@bitworking.org)"
__copyright__ = "Copyright 2006, Joe Gregorio"
__contributors__ = []
__version__ = "1.0.0"
__license__ = "MIT"
__history__ = """
"""
import urlparse
# Convert an IRI to a URI following the rules in RFC 3987
#
# The characters we need to enocde and escape are defined in the spec:
#
# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD
# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF
# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD
# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD
# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD
# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD
# / %xD0000-DFFFD / %xE1000-EFFFD
escape_range = [
(0xA0, 0xD7FF),
(0xE000, 0xF8FF),
(0xF900, 0xFDCF),
(0xFDF0, 0xFFEF),
(0x10000, 0x1FFFD),
(0x20000, 0x2FFFD),
(0x30000, 0x3FFFD),
(0x40000, 0x4FFFD),
(0x50000, 0x5FFFD),
(0x60000, 0x6FFFD),
(0x70000, 0x7FFFD),
(0x80000, 0x8FFFD),
(0x90000, 0x9FFFD),
(0xA0000, 0xAFFFD),
(0xB0000, 0xBFFFD),
(0xC0000, 0xCFFFD),
(0xD0000, 0xDFFFD),
(0xE1000, 0xEFFFD),
(0xF0000, 0xFFFFD),
(0x100000, 0x10FFFD),
]
def encode(c):
retval = c
i = ord(c)
for low, high in escape_range:
if i < low:
break
if i >= low and i <= high:
retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')])
break
return retval
def iri2uri(uri):
"""Convert an IRI to a URI. Note that IRIs must be
passed in a unicode strings. That is, do not utf-8 encode
the IRI before passing it into the function."""
if isinstance(uri ,unicode):
(scheme, authority, path, query, fragment) = urlparse.urlsplit(uri)
authority = authority.encode('idna')
# For each character in 'ucschar' or 'iprivate'
# 1. encode as utf-8
# 2. then %-encode each octet of that utf-8
uri = urlparse.urlunsplit((scheme, authority, path, query, fragment))
uri = "".join([encode(c) for c in uri])
return uri
if __name__ == "__main__":
import unittest
class Test(unittest.TestCase):
def test_uris(self):
"""Test that URIs are invariant under the transformation."""
invariant = [
u"ftp://ftp.is.co.za/rfc/rfc1808.txt",
u"http://www.ietf.org/rfc/rfc2396.txt",
u"ldap://[2001:db8::7]/c=GB?objectClass?one",
u"mailto:John.Doe@example.com",
u"news:comp.infosystems.www.servers.unix",
u"tel:+1-816-555-1212",
u"telnet://192.0.2.16:80/",
u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ]
for uri in invariant:
self.assertEqual(uri, iri2uri(uri))
def test_iri(self):
""" Test that the right type of escaping is done for each part of the URI."""
self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}"))
self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}"))
self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}"))
self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}"))
self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))
self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")))
self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8')))
unittest.main()
package/bin/da_ess_contentupdate/httplib2/socks.py
-438
View File
@@ -1,438 +0,0 @@
"""SocksiPy - Python SOCKS module.
Version 1.00
Copyright 2006 Dan-Haim. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of Dan Haim nor the names of his contributors may be used
to endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE.
This module provides a standard socket-like interface for Python
for tunneling connections through SOCKS proxies.
"""
"""
Minor modifications made by Christopher Gilbert (http://motomastyle.com/)
for use in PyLoris (http://pyloris.sourceforge.net/)
Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/)
mainly to merge bug fixes found in Sourceforge
"""
import base64
import socket
import struct
import sys
if getattr(socket, 'socket', None) is None:
raise ImportError('socket.socket missing, proxy support unusable')
PROXY_TYPE_SOCKS4 = 1
PROXY_TYPE_SOCKS5 = 2
PROXY_TYPE_HTTP = 3
PROXY_TYPE_HTTP_NO_TUNNEL = 4
_defaultproxy = None
_orgsocket = socket.socket
class ProxyError(Exception): pass
class GeneralProxyError(ProxyError): pass
class Socks5AuthError(ProxyError): pass
class Socks5Error(ProxyError): pass
class Socks4Error(ProxyError): pass
class HTTPError(ProxyError): pass
_generalerrors = ("success",
"invalid data",
"not connected",
"not available",
"bad proxy type",
"bad input")
_socks5errors = ("succeeded",
"general SOCKS server failure",
"connection not allowed by ruleset",
"Network unreachable",
"Host unreachable",
"Connection refused",
"TTL expired",
"Command not supported",
"Address type not supported",
"Unknown error")
_socks5autherrors = ("succeeded",
"authentication is required",
"all offered authentication methods were rejected",
"unknown username or invalid password",
"unknown error")
_socks4errors = ("request granted",
"request rejected or failed",
"request rejected because SOCKS server cannot connect to identd on the client",
"request rejected because the client program and identd report different user-ids",
"unknown error")
def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
"""setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
Sets a default proxy which all further socksocket objects will use,
unless explicitly changed.
"""
global _defaultproxy
_defaultproxy = (proxytype, addr, port, rdns, username, password)
def wrapmodule(module):
"""wrapmodule(module)
Attempts to replace a module's socket library with a SOCKS socket. Must set
a default proxy using setdefaultproxy(...) first.
This will only work on modules that import socket directly into the namespace;
most of the Python Standard Library falls into this category.
"""
if _defaultproxy != None:
module.socket.socket = socksocket
else:
raise GeneralProxyError((4, "no proxy specified"))
class socksocket(socket.socket):
"""socksocket([family[, type[, proto]]]) -> socket object
Open a SOCKS enabled socket. The parameters are the same as
those of the standard socket init. In order for SOCKS to work,
you must specify family=AF_INET, type=SOCK_STREAM and proto=0.
"""
def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None):
_orgsocket.__init__(self, family, type, proto, _sock)
if _defaultproxy != None:
self.__proxy = _defaultproxy
else:
self.__proxy = (None, None, None, None, None, None)
self.__proxysockname = None
self.__proxypeername = None
self.__httptunnel = True
def __recvall(self, count):
"""__recvall(count) -> data
Receive EXACTLY the number of bytes requested from the socket.
Blocks until the required number of bytes have been received.
"""
data = self.recv(count)
while len(data) < count:
d = self.recv(count-len(data))
if not d: raise GeneralProxyError((0, "connection closed unexpectedly"))
data = data + d
return data
def sendall(self, content, *args):
""" override socket.socket.sendall method to rewrite the header
for non-tunneling proxies if needed
"""
if not self.__httptunnel:
content = self.__rewriteproxy(content)
return super(socksocket, self).sendall(content, *args)
def __rewriteproxy(self, header):
""" rewrite HTTP request headers to support non-tunneling proxies
(i.e. those which do not support the CONNECT method).
This only works for HTTP (not HTTPS) since HTTPS requires tunneling.
"""
host, endpt = None, None
hdrs = header.split("\r\n")
for hdr in hdrs:
if hdr.lower().startswith("host:"):
host = hdr
elif hdr.lower().startswith("get") or hdr.lower().startswith("post"):
endpt = hdr
if host and endpt:
hdrs.remove(host)
hdrs.remove(endpt)
host = host.split(" ")[1]
endpt = endpt.split(" ")
if (self.__proxy[4] != None and self.__proxy[5] != None):
hdrs.insert(0, self.__getauthheader())
hdrs.insert(0, "Host: %s" % host)
hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2]))
return "\r\n".join(hdrs)
def __getauthheader(self):
auth = self.__proxy[4] + ":" + self.__proxy[5]
return "Proxy-Authorization: Basic " + base64.b64encode(auth)
def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
"""setproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
Sets the proxy to be used.
proxytype - The type of the proxy to be used. Three types
are supported: PROXY_TYPE_SOCKS4 (including socks4a),
PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP
addr - The address of the server (IP or DNS).
port - The port of the server. Defaults to 1080 for SOCKS
servers and 8080 for HTTP proxy servers.
rdns - Should DNS queries be preformed on the remote side
(rather than the local side). The default is True.
Note: This has no effect with SOCKS4 servers.
username - Username to authenticate with to the server.
The default is no authentication.
password - Password to authenticate with to the server.
Only relevant when username is also provided.
"""
self.__proxy = (proxytype, addr, port, rdns, username, password)
def __negotiatesocks5(self, destaddr, destport):
"""__negotiatesocks5(self,destaddr,destport)
Negotiates a connection through a SOCKS5 server.
"""
# First we'll send the authentication packages we support.
if (self.__proxy[4]!=None) and (self.__proxy[5]!=None):
# The username/password details were supplied to the
# setproxy method so we support the USERNAME/PASSWORD
# authentication (in addition to the standard none).
self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02))
else:
# No username/password were entered, therefore we
# only support connections with no authentication.
self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00))
# We'll receive the server's response to determine which
# method was selected
chosenauth = self.__recvall(2)
if chosenauth[0:1] != chr(0x05).encode():
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
# Check the chosen authentication method
if chosenauth[1:2] == chr(0x00).encode():
# No authentication is required
pass
elif chosenauth[1:2] == chr(0x02).encode():
# Okay, we need to perform a basic username/password
# authentication.
self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5])
authstat = self.__recvall(2)
if authstat[0:1] != chr(0x01).encode():
# Bad response
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
if authstat[1:2] != chr(0x00).encode():
# Authentication failed
self.close()
raise Socks5AuthError((3, _socks5autherrors[3]))
# Authentication succeeded
else:
# Reaching here is always bad
self.close()
if chosenauth[1] == chr(0xFF).encode():
raise Socks5AuthError((2, _socks5autherrors[2]))
else:
raise GeneralProxyError((1, _generalerrors[1]))
# Now we can request the actual connection
req = struct.pack('BBB', 0x05, 0x01, 0x00)
# If the given destination address is an IP address, we'll
# use the IPv4 address request even if remote resolving was specified.
try:
ipaddr = socket.inet_aton(destaddr)
req = req + chr(0x01).encode() + ipaddr
except socket.error:
# Well it's not an IP number, so it's probably a DNS name.
if self.__proxy[3]:
# Resolve remotely
ipaddr = None
req = req + chr(0x03).encode() + chr(len(destaddr)).encode() + destaddr
else:
# Resolve locally
ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
req = req + chr(0x01).encode() + ipaddr
req = req + struct.pack(">H", destport)
self.sendall(req)
# Get the response
resp = self.__recvall(4)
if resp[0:1] != chr(0x05).encode():
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
elif resp[1:2] != chr(0x00).encode():
# Connection failed
self.close()
if ord(resp[1:2])<=8:
raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])]))
else:
raise Socks5Error((9, _socks5errors[9]))
# Get the bound address/port
elif resp[3:4] == chr(0x01).encode():
boundaddr = self.__recvall(4)
elif resp[3:4] == chr(0x03).encode():
resp = resp + self.recv(1)
boundaddr = self.__recvall(ord(resp[4:5]))
else:
self.close()
raise GeneralProxyError((1,_generalerrors[1]))
boundport = struct.unpack(">H", self.__recvall(2))[0]
self.__proxysockname = (boundaddr, boundport)
if ipaddr != None:
self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
else:
self.__proxypeername = (destaddr, destport)
def getproxysockname(self):
"""getsockname() -> address info
Returns the bound IP address and port number at the proxy.
"""
return self.__proxysockname
def getproxypeername(self):
"""getproxypeername() -> address info
Returns the IP and port number of the proxy.
"""
return _orgsocket.getpeername(self)
def getpeername(self):
"""getpeername() -> address info
Returns the IP address and port number of the destination
machine (note: getproxypeername returns the proxy)
"""
return self.__proxypeername
def __negotiatesocks4(self,destaddr,destport):
"""__negotiatesocks4(self,destaddr,destport)
Negotiates a connection through a SOCKS4 server.
"""
# Check if the destination address provided is an IP address
rmtrslv = False
try:
ipaddr = socket.inet_aton(destaddr)
except socket.error:
# It's a DNS name. Check where it should be resolved.
if self.__proxy[3]:
ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01)
rmtrslv = True
else:
ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
# Construct the request packet
req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr
# The username parameter is considered userid for SOCKS4
if self.__proxy[4] != None:
req = req + self.__proxy[4]
req = req + chr(0x00).encode()
# DNS name if remote resolving is required
# NOTE: This is actually an extension to the SOCKS4 protocol
# called SOCKS4A and may not be supported in all cases.
if rmtrslv:
req = req + destaddr + chr(0x00).encode()
self.sendall(req)
# Get the response from the server
resp = self.__recvall(8)
if resp[0:1] != chr(0x00).encode():
# Bad data
self.close()
raise GeneralProxyError((1,_generalerrors[1]))
if resp[1:2] != chr(0x5A).encode():
# Server returned an error
self.close()
if ord(resp[1:2]) in (91, 92, 93):
self.close()
raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90]))
else:
raise Socks4Error((94, _socks4errors[4]))
# Get the bound address/port
self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0])
if rmtrslv != None:
self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
else:
self.__proxypeername = (destaddr, destport)
def __negotiatehttp(self, destaddr, destport):
"""__negotiatehttp(self,destaddr,destport)
Negotiates a connection through an HTTP server.
"""
# If we need to resolve locally, we do this now
if not self.__proxy[3]:
addr = socket.gethostbyname(destaddr)
else:
addr = destaddr
headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"]
headers += ["Host: ", destaddr, "\r\n"]
if (self.__proxy[4] != None and self.__proxy[5] != None):
headers += [self.__getauthheader(), "\r\n"]
headers.append("\r\n")
self.sendall("".join(headers).encode())
# We read the response until we get the string "\r\n\r\n"
resp = self.recv(1)
while resp.find("\r\n\r\n".encode()) == -1:
resp = resp + self.recv(1)
# We just need the first line to check if the connection
# was successful
statusline = resp.splitlines()[0].split(" ".encode(), 2)
if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()):
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
try:
statuscode = int(statusline[1])
except ValueError:
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
if statuscode != 200:
self.close()
raise HTTPError((statuscode, statusline[2]))
self.__proxysockname = ("0.0.0.0", 0)
self.__proxypeername = (addr, destport)
def connect(self, destpair):
"""connect(self, despair)
Connects to the specified destination through a proxy.
destpar - A tuple of the IP/DNS address and the port number.
(identical to socket's connect).
To select the proxy server use setproxy().
"""
# Do a minimal input check first
if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (not isinstance(destpair[0], basestring)) or (type(destpair[1]) != int):
raise GeneralProxyError((5, _generalerrors[5]))
if self.__proxy[0] == PROXY_TYPE_SOCKS5:
if self.__proxy[2] != None:
portnum = self.__proxy[2]
else:
portnum = 1080
_orgsocket.connect(self, (self.__proxy[1], portnum))
self.__negotiatesocks5(destpair[0], destpair[1])
elif self.__proxy[0] == PROXY_TYPE_SOCKS4:
if self.__proxy[2] != None:
portnum = self.__proxy[2]
else:
portnum = 1080
_orgsocket.connect(self,(self.__proxy[1], portnum))
self.__negotiatesocks4(destpair[0], destpair[1])
elif self.__proxy[0] == PROXY_TYPE_HTTP:
if self.__proxy[2] != None:
portnum = self.__proxy[2]
else:
portnum = 8080
_orgsocket.connect(self,(self.__proxy[1], portnum))
self.__negotiatehttp(destpair[0], destpair[1])
elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL:
if self.__proxy[2] != None:
portnum = self.__proxy[2]
else:
portnum = 8080
_orgsocket.connect(self,(self.__proxy[1],portnum))
if destpair[1] == 443:
self.__negotiatehttp(destpair[0],destpair[1])
else:
self.__httptunnel = False
elif self.__proxy[0] == None:
_orgsocket.connect(self, (destpair[0], destpair[1]))
else:
raise GeneralProxyError((4, _generalerrors[4]))
package/bin/da_ess_contentupdate/jinja2/__init__.py
-83
View File
@@ -1,83 +0,0 @@
# -*- coding: utf-8 -*-
"""
jinja2
~~~~~~
Jinja2 is a template engine written in pure Python. It provides a
Django inspired non-XML syntax but supports inline expressions and
an optional sandboxed environment.
Nutshell
--------
Here a small example of a Jinja2 template::
{% extends 'base.html' %}
{% block title %}Memberlist{% endblock %}
{% block content %}
<ul>
{% for user in users %}
<li><a href="{{ user.url }}">{{ user.username }}</a></li>
{% endfor %}
</ul>
{% endblock %}
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
__docformat__ = 'restructuredtext en'
__version__ = "2.10.3"
# high level interface
from jinja2.environment import Environment, Template
# loaders
from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \
DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \
ModuleLoader
# bytecode caches
from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \
MemcachedBytecodeCache
# undefined types
from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined, \
make_logging_undefined
# exceptions
from jinja2.exceptions import TemplateError, UndefinedError, \
TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \
TemplateAssertionError, TemplateRuntimeError
# decorators and public utilities
from jinja2.filters import environmentfilter, contextfilter, \
evalcontextfilter
from jinja2.utils import Markup, escape, clear_caches, \
environmentfunction, evalcontextfunction, contextfunction, \
is_undefined, select_autoescape
__all__ = [
'Environment', 'Template', 'BaseLoader', 'FileSystemLoader',
'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader',
'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache',
'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined',
'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound',
'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError',
'TemplateRuntimeError',
'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape',
'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined',
'evalcontextfilter', 'evalcontextfunction', 'make_logging_undefined',
'select_autoescape',
]
def _patch_async():
from jinja2.utils import have_async_gen
if have_async_gen:
from jinja2.asyncsupport import patch_all
patch_all()
_patch_async()
del _patch_async
package/bin/da_ess_contentupdate/jinja2/_compat.py
-105
View File
@@ -1,105 +0,0 @@
# -*- coding: utf-8 -*-
"""
jinja2._compat
~~~~~~~~~~~~~~
Some py2/py3 compatibility support based on a stripped down
version of six so we don't have to depend on a specific version
of it.
:copyright: Copyright 2013 by the Jinja team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import sys
PY2 = sys.version_info[0] == 2
PYPY = hasattr(sys, 'pypy_translation_info')
_identity = lambda x: x
if not PY2:
unichr = chr
range_type = range
text_type = str
string_types = (str,)
integer_types = (int,)
iterkeys = lambda d: iter(d.keys())
itervalues = lambda d: iter(d.values())
iteritems = lambda d: iter(d.items())
import pickle
from io import BytesIO, StringIO
NativeStringIO = StringIO
def reraise(tp, value, tb=None):
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
ifilter = filter
imap = map
izip = zip
intern = sys.intern
implements_iterator = _identity
implements_to_string = _identity
encode_filename = _identity
else:
unichr = unichr
text_type = unicode
range_type = xrange
string_types = (str, unicode)
integer_types = (int, long)
iterkeys = lambda d: d.iterkeys()
itervalues = lambda d: d.itervalues()
iteritems = lambda d: d.iteritems()
import cPickle as pickle
from cStringIO import StringIO as BytesIO, StringIO
NativeStringIO = BytesIO
exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')
from itertools import imap, izip, ifilter
intern = intern
def implements_iterator(cls):
cls.next = cls.__next__
del cls.__next__
return cls
def implements_to_string(cls):
cls.__unicode__ = cls.__str__
cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
return cls
def encode_filename(filename):
if isinstance(filename, unicode):
return filename.encode('utf-8')
return filename
def with_metaclass(meta, *bases):
"""Create a base class with a metaclass."""
# This requires a bit of explanation: the basic idea is to make a
# dummy metaclass for one level of class instantiation that replaces
# itself with the actual metaclass.
class metaclass(type):
def __new__(cls, name, this_bases, d):
return meta(name, bases, d)
return type.__new__(metaclass, 'temporary_class', (), {})
try:
from urllib.parse import quote_from_bytes as url_quote
except ImportError:
from urllib import quote as url_quote
try:
from collections import abc
except ImportError:
import collections as abc
package/bin/da_ess_contentupdate/jinja2/_identifier.py
-2
View File
@@ -1,2 +0,0 @@
# generated by scripts/generate_identifier_pattern.py
pattern = '·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯'
package/bin/da_ess_contentupdate/jinja2/asyncfilters.py
-146
View File
@@ -1,146 +0,0 @@
from functools import wraps
from jinja2.asyncsupport import auto_aiter
from jinja2 import filters
async def auto_to_seq(value):
seq = []
if hasattr(value, '__aiter__'):
async for item in value:
seq.append(item)
else:
for item in value:
seq.append(item)
return seq
async def async_select_or_reject(args, kwargs, modfunc, lookup_attr):
seq, func = filters.prepare_select_or_reject(
args, kwargs, modfunc, lookup_attr)
if seq:
async for item in auto_aiter(seq):
if func(item):
yield item
def dualfilter(normal_filter, async_filter):
wrap_evalctx = False
if getattr(normal_filter, 'environmentfilter', False):
is_async = lambda args: args[0].is_async
wrap_evalctx = False
else:
if not getattr(normal_filter, 'evalcontextfilter', False) and \
not getattr(normal_filter, 'contextfilter', False):
wrap_evalctx = True
is_async = lambda args: args[0].environment.is_async
@wraps(normal_filter)
def wrapper(*args, **kwargs):
b = is_async(args)
if wrap_evalctx:
args = args[1:]
if b:
return async_filter(*args, **kwargs)
return normal_filter(*args, **kwargs)
if wrap_evalctx:
wrapper.evalcontextfilter = True
wrapper.asyncfiltervariant = True
return wrapper
def asyncfiltervariant(original):
def decorator(f):
return dualfilter(original, f)
return decorator
@asyncfiltervariant(filters.do_first)
async def do_first(environment, seq):
try:
return await auto_aiter(seq).__anext__()
except StopAsyncIteration:
return environment.undefined('No first item, sequence was empty.')
@asyncfiltervariant(filters.do_groupby)
async def do_groupby(environment, value, attribute):
expr = filters.make_attrgetter(environment, attribute)
return [filters._GroupTuple(key, await auto_to_seq(values))
for key, values in filters.groupby(sorted(
await auto_to_seq(value), key=expr), expr)]
@asyncfiltervariant(filters.do_join)
async def do_join(eval_ctx, value, d=u'', attribute=None):
return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute)
@asyncfiltervariant(filters.do_list)
async def do_list(value):
return await auto_to_seq(value)
@asyncfiltervariant(filters.do_reject)
async def do_reject(*args, **kwargs):
return async_select_or_reject(args, kwargs, lambda x: not x, False)
@asyncfiltervariant(filters.do_rejectattr)
async def do_rejectattr(*args, **kwargs):
return async_select_or_reject(args, kwargs, lambda x: not x, True)
@asyncfiltervariant(filters.do_select)
async def do_select(*args, **kwargs):
return async_select_or_reject(args, kwargs, lambda x: x, False)
@asyncfiltervariant(filters.do_selectattr)
async def do_selectattr(*args, **kwargs):
return async_select_or_reject(args, kwargs, lambda x: x, True)
@asyncfiltervariant(filters.do_map)
async def do_map(*args, **kwargs):
seq, func = filters.prepare_map(args, kwargs)
if seq:
async for item in auto_aiter(seq):
yield func(item)
@asyncfiltervariant(filters.do_sum)
async def do_sum(environment, iterable, attribute=None, start=0):
rv = start
if attribute is not None:
func = filters.make_attrgetter(environment, attribute)
else:
func = lambda x: x
async for item in auto_aiter(iterable):
rv += func(item)
return rv
@asyncfiltervariant(filters.do_slice)
async def do_slice(value, slices, fill_with=None):
return filters.do_slice(await auto_to_seq(value), slices, fill_with)
ASYNC_FILTERS = {
'first': do_first,
'groupby': do_groupby,
'join': do_join,
'list': do_list,
# we intentionally do not support do_last because that would be
# ridiculous
'reject': do_reject,
'rejectattr': do_rejectattr,
'map': do_map,
'select': do_select,
'selectattr': do_selectattr,
'sum': do_sum,
'slice': do_slice,
}
package/bin/da_ess_contentupdate/jinja2/asyncsupport.py
-256
View File
@@ -1,256 +0,0 @@
# -*- coding: utf-8 -*-
"""
jinja2.asyncsupport
~~~~~~~~~~~~~~~~~~~
Has all the code for async support which is implemented as a patch
for supported Python versions.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
import sys
import asyncio
import inspect
from functools import update_wrapper
from jinja2.utils import concat, internalcode, Markup
from jinja2.environment import TemplateModule
from jinja2.runtime import LoopContextBase, _last_iteration
async def concat_async(async_gen):
rv = []
async def collect():
async for event in async_gen:
rv.append(event)
await collect()
return concat(rv)
async def generate_async(self, *args, **kwargs):
vars = dict(*args, **kwargs)
try:
async for event in self.root_render_func(self.new_context(vars)):
yield event
except Exception:
exc_info = sys.exc_info()
else:
return
yield self.environment.handle_exception(exc_info, True)
def wrap_generate_func(original_generate):
def _convert_generator(self, loop, args, kwargs):
async_gen = self.generate_async(*args, **kwargs)
try:
while 1:
yield loop.run_until_complete(async_gen.__anext__())
except StopAsyncIteration:
pass
def generate(self, *args, **kwargs):
if not self.environment.is_async:
return original_generate(self, *args, **kwargs)
return _convert_generator(self, asyncio.get_event_loop(), args, kwargs)
return update_wrapper(generate, original_generate)
async def render_async(self, *args, **kwargs):
if not self.environment.is_async:
raise RuntimeError('The environment was not created with async mode '
'enabled.')
vars = dict(*args, **kwargs)
ctx = self.new_context(vars)
try:
return await concat_async(self.root_render_func(ctx))
except Exception:
exc_info = sys.exc_info()
return self.environment.handle_exception(exc_info, True)
def wrap_render_func(original_render):
def render(self, *args, **kwargs):
if not self.environment.is_async:
return original_render(self, *args, **kwargs)
loop = asyncio.get_event_loop()
return loop.run_until_complete(self.render_async(*args, **kwargs))
return update_wrapper(render, original_render)
def wrap_block_reference_call(original_call):
@internalcode
async def async_call(self):
rv = await concat_async(self._stack[self._depth](self._context))
if self._context.eval_ctx.autoescape:
rv = Markup(rv)
return rv
@internalcode
def __call__(self):
if not self._context.environment.is_async:
return original_call(self)
return async_call(self)
return update_wrapper(__call__, original_call)
def wrap_macro_invoke(original_invoke):
@internalcode
async def async_invoke(self, arguments, autoescape):
rv = await self._func(*arguments)
if autoescape:
rv = Markup(rv)
return rv
@internalcode
def _invoke(self, arguments, autoescape):
if not self._environment.is_async:
return original_invoke(self, arguments, autoescape)
return async_invoke(self, arguments, autoescape)
return update_wrapper(_invoke, original_invoke)
@internalcode
async def get_default_module_async(self):
if self._module is not None:
return self._module
self._module = rv = await self.make_module_async()
return rv
def wrap_default_module(original_default_module):
@internalcode
def _get_default_module(self):
if self.environment.is_async:
raise RuntimeError('Template module attribute is unavailable '
'in async mode')
return original_default_module(self)
return _get_default_module
async def make_module_async(self, vars=None, shared=False, locals=None):
context = self.new_context(vars, shared, locals)
body_stream = []
async for item in self.root_render_func(context):
body_stream.append(item)
return TemplateModule(self, context, body_stream)
def patch_template():
from jinja2 import Template
Template.generate = wrap_generate_func(Template.generate)
Template.generate_async = update_wrapper(
generate_async, Template.generate_async)
Template.render_async = update_wrapper(
render_async, Template.render_async)
Template.render = wrap_render_func(Template.render)
Template._get_default_module = wrap_default_module(
Template._get_default_module)
Template._get_default_module_async = get_default_module_async
Template.make_module_async = update_wrapper(
make_module_async, Template.make_module_async)
def patch_runtime():
from jinja2.runtime import BlockReference, Macro
BlockReference.__call__ = wrap_block_reference_call(
BlockReference.__call__)
Macro._invoke = wrap_macro_invoke(Macro._invoke)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def patch_all():
patch_template()
patch_runtime()
patch_filters()
async def auto_await(value):
if inspect.isawaitable(value):
return await value
return value
async def auto_aiter(iterable):
if hasattr(iterable, '__aiter__'):
async for item in iterable:
yield item
return
for item in iterable:
yield item
class AsyncLoopContext(LoopContextBase):
def __init__(self, async_iterator, undefined, after, length, recurse=None,
depth0=0):
LoopContextBase.__init__(self, undefined, recurse, depth0)
self._async_iterator = async_iterator
self._after = after
self._length = length
@property
def length(self):
if self._length is None:
raise TypeError('Loop length for some iterators cannot be '
'lazily calculated in async mode')
return self._length
def __aiter__(self):
return AsyncLoopContextIterator(self)
class AsyncLoopContextIterator(object):
__slots__ = ('context',)
def __init__(self, context):
self.context = context
def __aiter__(self):
return self
async def __anext__(self):
ctx = self.context
ctx.index0 += 1
if ctx._after is _last_iteration:
raise StopAsyncIteration()
ctx._before = ctx._current
ctx._current = ctx._after
try:
ctx._after = await ctx._async_iterator.__anext__()
except StopAsyncIteration:
ctx._after = _last_iteration
return ctx._current, ctx
async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0):
# Length is more complicated and less efficient in async mode. The
# reason for this is that we cannot know if length will be used
# upfront but because length is a property we cannot lazily execute it
# later. This means that we need to buffer it up and measure :(
#
# We however only do this for actual iterators, not for async
# iterators as blocking here does not seem like the best idea in the
# world.
try:
length = len(iterable)
except (TypeError, AttributeError):
if not hasattr(iterable, '__aiter__'):
iterable = tuple(iterable)
length = len(iterable)
else:
length = None
async_iterator = auto_aiter(iterable)
try:
after = await async_iterator.__anext__()
except StopAsyncIteration:
after = _last_iteration
return AsyncLoopContext(async_iterator, undefined, after, length, recurse,
depth0)
package/bin/da_ess_contentupdate/jinja2/bccache.py
-361
View File
@@ -1,361 +0,0 @@
# -*- coding: utf-8 -*-
"""
jinja2.bccache
~~~~~~~~~~~~~~
This module implements the bytecode cache system Jinja is optionally
using. This is useful if you have very complex template situations and
the compiliation of all those templates slow down your application too
much.
Situations where this is useful are often forking web applications that
are initialized on the first request.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD.
"""
from os import path, listdir
import os
import sys
import stat
import errno
import marshal
import tempfile
import fnmatch
from hashlib import sha1
from jinja2.utils import open_if_exists
from jinja2._compat import BytesIO, pickle, PY2, text_type
# marshal works better on 3.x, one hack less required
if not PY2:
marshal_dump = marshal.dump
marshal_load = marshal.load
else:
def marshal_dump(code, f):
if isinstance(f, file):
marshal.dump(code, f)
else:
f.write(marshal.dumps(code))
def marshal_load(f):
if isinstance(f, file):
return marshal.load(f)
return marshal.loads(f.read())
bc_version = 3
# magic version used to only change with new jinja versions. With 2.6
# we change this to also take Python version changes into account. The
# reason for this is that Python tends to segfault if fed earlier bytecode
# versions because someone thought it would be a good idea to reuse opcodes
# or make Python incompatible with earlier versions.
bc_magic = 'j2'.encode('ascii') + \
pickle.dumps(bc_version, 2) + \
pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1])
class Bucket(object):
"""Buckets are used to store the bytecode for one template. It's created
and initialized by the bytecode cache and passed to the loading functions.
The buckets get an internal checksum from the cache assigned and use this
to automatically reject outdated cache material. Individual bytecode
cache subclasses don't have to care about cache invalidation.
"""
def __init__(self, environment, key, checksum):
self.environment = environment
self.key = key
self.checksum = checksum
self.reset()
def reset(self):
"""Resets the bucket (unloads the bytecode)."""
self.code = None
def load_bytecode(self, f):
"""Loads bytecode from a file or file like object."""
# make sure the magic header is correct
magic = f.read(len(bc_magic))
if magic != bc_magic:
self.reset()
return
# the source code of the file changed, we need to reload
checksum = pickle.load(f)
if self.checksum != checksum:
self.reset()
return
# if marshal_load fails then we need to reload
try:
self.code = marshal_load(f)
except (EOFError, ValueError, TypeError):
self.reset()
return
def write_bytecode(self, f):
"""Dump the bytecode into the file or file like object passed."""
if self.code is None:
raise TypeError('can\'t write empty bucket')
f.write(bc_magic)
pickle.dump(self.checksum, f, 2)
marshal_dump(self.code, f)
def bytecode_from_string(self, string):
"""Load bytecode from a string."""
self.load_bytecode(BytesIO(string))
def bytecode_to_string(self):
"""Return the bytecode as string."""
out = BytesIO()
self.write_bytecode(out)
return out.getvalue()
class BytecodeCache(object):
"""To implement your own bytecode cache you have to subclass this class
and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of
these methods are passed a :class:`~jinja2.bccache.Bucket`.
A very basic bytecode cache that saves the bytecode on the file system::
from os import path
class MyCache(BytecodeCache):
def __init__(self, directory):
self.directory = directory
def load_bytecode(self, bucket):
filename = path.join(self.directory, bucket.key)
if path.exists(filename):
with open(filename, 'rb') as f:
bucket.load_bytecode(f)
def dump_bytecode(self, bucket):
filename = path.join(self.directory, bucket.key)
with open(filename, 'wb') as f:
bucket.write_bytecode(f)
A more advanced version of a filesystem based bytecode cache is part of
Jinja2.
"""
def load_bytecode(self, bucket):
"""Subclasses have to override this method to load bytecode into a
bucket. If they are not able to find code in the cache for the
bucket, it must not do anything.
"""
raise NotImplementedError()
def dump_bytecode(self, bucket):
"""Subclasses have to override this method to write the bytecode
from a bucket back to the cache. If it unable to do so it must not
fail silently but raise an exception.
"""
raise NotImplementedError()
def clear(self):
"""Clears the cache. This method is not used by Jinja2 but should be
implemented to allow applications to clear the bytecode cache used
by a particular environment.
"""
def get_cache_key(self, name, filename=None):
"""Returns the unique hash key for this template name."""
hash = sha1(name.encode('utf-8'))
if filename is not None:
filename = '|' + filename
if isinstance(filename, text_type):
filename = filename.encode('utf-8')
hash.update(filename)
return hash.hexdigest()
def get_source_checksum(self, source):
"""Returns a checksum for the source."""
return sha1(source.encode('utf-8')).hexdigest()
def get_bucket(self, environment, name, filename, source):
"""Return a cache bucket for the given template. All arguments are
mandatory but filename may be `None`.
"""
key = self.get_cache_key(name, filename)
checksum = self.get_source_checksum(source)
bucket = Bucket(environment, key, checksum)
self.load_bytecode(bucket)
return bucket
def set_bucket(self, bucket):
"""Put the bucket into the cache."""
self.dump_bytecode(bucket)
class FileSystemBytecodeCache(BytecodeCache):
"""A bytecode cache that stores bytecode on the filesystem. It accepts
two arguments: The directory where the cache items are stored and a
pattern string that is used to build the filename.
If no directory is specified a default cache directory is selected. On
Windows the user's temp directory is used, on UNIX systems a directory
is created for the user in the system temp directory.
The pattern can be used to have multiple separate caches operate on the
same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s``
is replaced with the cache key.
>>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
This bytecode cache supports clearing of the cache using the clear method.
"""
def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
if directory is None:
directory = self._get_default_cache_dir()
self.directory = directory
self.pattern = pattern
def _get_default_cache_dir(self):
def _unsafe_dir():
raise RuntimeError('Cannot determine safe temp directory. You '
'need to explicitly provide one.')
tmpdir = tempfile.gettempdir()
# On windows the temporary directory is used specific unless
# explicitly forced otherwise. We can just use that.
if os.name == 'nt':
return tmpdir
if not hasattr(os, 'getuid'):
_unsafe_dir()
dirname = '_jinja2-cache-%d' % os.getuid()
actual_dir = os.path.join(tmpdir, dirname)
try:
os.mkdir(actual_dir, stat.S_IRWXU)
except OSError as e:
if e.errno != errno.EEXIST:
raise
try:
os.chmod(actual_dir, stat.S_IRWXU)
actual_dir_stat = os.lstat(actual_dir)
if actual_dir_stat.st_uid != os.getuid() \
or not stat.S_ISDIR(actual_dir_stat.st_mode) \
or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU:
_unsafe_dir()
except OSError as e:
if e.errno != errno.EEXIST:
raise
actual_dir_stat = os.lstat(actual_dir)
if actual_dir_stat.st_uid != os.getuid() \
or not stat.S_ISDIR(actual_dir_stat.st_mode) \
or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU:
_unsafe_dir()
return actual_dir
def _get_cache_filename(self, bucket):
return path.join(self.directory, self.pattern % bucket.key)
def load_bytecode(self, bucket):
f = open_if_exists(self._get_cache_filename(bucket), 'rb')
if f is not None:
try:
bucket.load_bytecode(f)
finally:
f.close()
def dump_bytecode(self, bucket):
f = open(self._get_cache_filename(bucket), 'wb')
try:
bucket.write_bytecode(f)
finally:
f.close()
def clear(self):
# imported lazily here because google app-engine doesn't support
# write access on the file system and the function does not exist
# normally.
from os import remove
files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
for filename in files:
try:
remove(path.join(self.directory, filename))
except OSError:
pass
class MemcachedBytecodeCache(BytecodeCache):
"""This class implements a bytecode cache that uses a memcache cache for
storing the information. It does not enforce a specific memcache library
(tummy's memcache or cmemcache) but will accept any class that provides
the minimal interface required.
Libraries compatible with this class:
- `cachelib <https://github.com/pallets/cachelib>`_
- `python-memcached <https://pypi.org/project/python-memcached/>`_
(Unfortunately the django cache interface is not compatible because it
does not support storing binary data, only unicode. You can however pass
the underlying cache client to the bytecode cache which is available
as `django.core.cache.cache._client`.)
The minimal interface for the client passed to the constructor is this:
.. class:: MinimalClientInterface
.. method:: set(key, value[, timeout])
Stores the bytecode in the cache. `value` is a string and
`timeout` the timeout of the key. If timeout is not provided
a default timeout or no timeout should be assumed, if it's
provided it's an integer with the number of seconds the cache
item should exist.
.. method:: get(key)
Returns the value for the cache key. If the item does not
exist in the cache the return value must be `None`.
The other arguments to the constructor are the prefix for all keys that
is added before the actual cache key and the timeout for the bytecode in
the cache system. We recommend a high (or no) timeout.
This bytecode cache does not support clearing of used items in the cache.
The clear method is a no-operation function.
.. versionadded:: 2.7
Added support for ignoring memcache errors through the
`ignore_memcache_errors` parameter.
"""
def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
ignore_memcache_errors=True):
self.client = client
self.prefix = prefix
self.timeout = timeout
self.ignore_memcache_errors = ignore_memcache_errors
def load_bytecode(self, bucket):
try:
code = self.client.get(self.prefix + bucket.key)
except Exception:
if not self.ignore_memcache_errors:
raise
code = None
if code is not None:
bucket.bytecode_from_string(code)
def dump_bytecode(self, bucket):
args = (self.prefix + bucket.key, bucket.bytecode_to_string())
if self.timeout is not None:
args += (self.timeout,)
try:
self.client.set(*args)
except Exception:
if not self.ignore_memcache_errors:
raise
package/bin/da_ess_contentupdate/jinja2/compiler.py
-1721
View File
File diff suppressed because it is too large Load Diff
package/bin/da_ess_contentupdate/jinja2/constants.py
-32
View File
@@ -1,32 +0,0 @@
# -*- coding: utf-8 -*-
"""
jinja.constants
~~~~~~~~~~~~~~~
Various constants.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
#: list of lorem ipsum words used by the lipsum() helper function
LOREM_IPSUM_WORDS = u'''\
a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at
auctor augue bibendum blandit class commodo condimentum congue consectetuer
consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus
diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend
elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames
faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac
hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum
justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem
luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie
mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non
nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque
penatibus per pharetra phasellus placerat platea porta porttitor posuere
potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus
ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit
sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor
tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices
ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus
viverra volutpat vulputate'''
package/bin/da_ess_contentupdate/jinja2/debug.py
-378
View File
@@ -1,378 +0,0 @@
# -*- coding: utf-8 -*-
"""
jinja2.debug
~~~~~~~~~~~~
Implements the debug interface for Jinja. This module does some pretty
ugly stuff with the Python traceback system in order to achieve tracebacks
with correct line numbers, locals and contents.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
import sys
import traceback
from types import TracebackType, CodeType
from jinja2.utils import missing, internal_code
from jinja2.exceptions import TemplateSyntaxError
from jinja2._compat import iteritems, reraise, PY2
# on pypy we can take advantage of transparent proxies
try:
from __pypy__ import tproxy
except ImportError:
tproxy = None
# how does the raise helper look like?
try:
exec("raise TypeError, 'foo'")
except SyntaxError:
raise_helper = 'raise __jinja_exception__[1]'
except TypeError:
raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]'
class TracebackFrameProxy(object):
"""Proxies a traceback frame."""
def __init__(self, tb):
self.tb = tb
self._tb_next = None
@property
def tb_next(self):
return self._tb_next
def set_next(self, next):
if tb_set_next is not None:
try:
tb_set_next(self.tb, next and next.tb or None)
except Exception:
# this function can fail due to all the hackery it does
# on various python implementations. We just catch errors
# down and ignore them if necessary.
pass
self._tb_next = next
@property
def is_jinja_frame(self):
return '__jinja_template__' in self.tb.tb_frame.f_globals
def __getattr__(self, name):
return getattr(self.tb, name)
def make_frame_proxy(frame):
proxy = TracebackFrameProxy(frame)
if tproxy is None:
return proxy
def operation_handler(operation, *args, **kwargs):
if operation in ('__getattribute__', '__getattr__'):
return getattr(proxy, args[0])
elif operation == '__setattr__':
proxy.__setattr__(*args, **kwargs)
else:
return getattr(proxy, operation)(*args, **kwargs)
return tproxy(TracebackType, operation_handler)
class ProcessedTraceback(object):
"""Holds a Jinja preprocessed traceback for printing or reraising."""
def __init__(self, exc_type, exc_value, frames):
assert frames, 'no frames for this traceback?'
self.exc_type = exc_type
self.exc_value = exc_value
self.frames = frames
# newly concatenate the frames (which are proxies)
prev_tb = None
for tb in self.frames:
if prev_tb is not None:
prev_tb.set_next(tb)
prev_tb = tb
prev_tb.set_next(None)
def render_as_text(self, limit=None):
"""Return a string with the traceback."""
lines = traceback.format_exception(self.exc_type, self.exc_value,
self.frames[0], limit=limit)
return ''.join(lines).rstrip()
def render_as_html(self, full=False):
"""Return a unicode string with the traceback as rendered HTML."""
from jinja2.debugrenderer import render_traceback
return u'%s\n\n<!--\n%s\n-->' % (
render_traceback(self, full=full),
self.render_as_text().decode('utf-8', 'replace')
)
@property
def is_template_syntax_error(self):
"""`True` if this is a template syntax error."""
return isinstance(self.exc_value, TemplateSyntaxError)
@property
def exc_info(self):
"""Exception info tuple with a proxy around the frame objects."""
return self.exc_type, self.exc_value, self.frames[0]
@property
def standard_exc_info(self):
"""Standard python exc_info for re-raising"""
tb = self.frames[0]
# the frame will be an actual traceback (or transparent proxy) if
# we are on pypy or a python implementation with support for tproxy
if type(tb) is not TracebackType:
tb = tb.tb
return self.exc_type, self.exc_value, tb
def make_traceback(exc_info, source_hint=None):
"""Creates a processed traceback object from the exc_info."""
exc_type, exc_value, tb = exc_info
if isinstance(exc_value, TemplateSyntaxError):
exc_info = translate_syntax_error(exc_value, source_hint)
initial_skip = 0
else:
initial_skip = 1
return translate_exception(exc_info, initial_skip)
def translate_syntax_error(error, source=None):
"""Rewrites a syntax error to please traceback systems."""
error.source = source
error.translated = True
exc_info = (error.__class__, error, None)
filename = error.filename
if filename is None:
filename = '<unknown>'
return fake_exc_info(exc_info, filename, error.lineno)
def translate_exception(exc_info, initial_skip=0):
"""If passed an exc_info it will automatically rewrite the exceptions
all the way down to the correct line numbers and frames.
"""
tb = exc_info[2]
frames = []
# skip some internal frames if wanted
for x in range(initial_skip):
if tb is not None:
tb = tb.tb_next
initial_tb = tb
while tb is not None:
# skip frames decorated with @internalcode. These are internal
# calls we can't avoid and that are useless in template debugging
# output.
if tb.tb_frame.f_code in internal_code:
tb = tb.tb_next
continue
# save a reference to the next frame if we override the current
# one with a faked one.
next = tb.tb_next
# fake template exceptions
template = tb.tb_frame.f_globals.get('__jinja_template__')
if template is not None:
lineno = template.get_corresponding_lineno(tb.tb_lineno)
tb = fake_exc_info(exc_info[:2] + (tb,), template.filename,
lineno)[2]
frames.append(make_frame_proxy(tb))
tb = next
# if we don't have any exceptions in the frames left, we have to
# reraise it unchanged.
# XXX: can we backup here? when could this happen?
if not frames:
reraise(exc_info[0], exc_info[1], exc_info[2])
return ProcessedTraceback(exc_info[0], exc_info[1], frames)
def get_jinja_locals(real_locals):
ctx = real_locals.get('context')
if ctx:
locals = ctx.get_all().copy()
else:
locals = {}
local_overrides = {}
for name, value in iteritems(real_locals):
if not name.startswith('l_') or value is missing:
continue
try:
_, depth, name = name.split('_', 2)
depth = int(depth)
except ValueError:
continue
cur_depth = local_overrides.get(name, (-1,))[0]
if cur_depth < depth:
local_overrides[name] = (depth, value)
for name, (_, value) in iteritems(local_overrides):
if value is missing:
locals.pop(name, None)
else:
locals[name] = value
return locals
def fake_exc_info(exc_info, filename, lineno):
"""Helper for `translate_exception`."""
exc_type, exc_value, tb = exc_info
# figure the real context out
if tb is not None:
locals = get_jinja_locals(tb.tb_frame.f_locals)
# if there is a local called __jinja_exception__, we get
# rid of it to not break the debug functionality.
locals.pop('__jinja_exception__', None)
else:
locals = {}
# assamble fake globals we need
globals = {
'__name__': filename,
'__file__': filename,
'__jinja_exception__': exc_info[:2],
# we don't want to keep the reference to the template around
# to not cause circular dependencies, but we mark it as Jinja
# frame for the ProcessedTraceback
'__jinja_template__': None
}
# and fake the exception
code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec')
# if it's possible, change the name of the code. This won't work
# on some python environments such as google appengine
try:
if tb is None:
location = 'template'
else:
function = tb.tb_frame.f_code.co_name
if function == 'root':
location = 'top-level template code'
elif function.startswith('block_'):
location = 'block "%s"' % function[6:]
else:
location = 'template'
if PY2:
code = CodeType(0, code.co_nlocals, code.co_stacksize,
code.co_flags, code.co_code, code.co_consts,
code.co_names, code.co_varnames, filename,
location, code.co_firstlineno,
code.co_lnotab, (), ())
else:
code = CodeType(0, code.co_kwonlyargcount,
code.co_nlocals, code.co_stacksize,
code.co_flags, code.co_code, code.co_consts,
code.co_names, code.co_varnames, filename,
location, code.co_firstlineno,
code.co_lnotab, (), ())
except Exception as e:
pass
# execute the code and catch the new traceback
try:
exec(code, globals, locals)
except:
exc_info = sys.exc_info()
new_tb = exc_info[2].tb_next
# return without this frame
return exc_info[:2] + (new_tb,)
def _init_ugly_crap():
"""This function implements a few ugly things so that we can patch the
traceback objects. The function returned allows resetting `tb_next` on
any python traceback object. Do not attempt to use this on non cpython
interpreters
"""
import ctypes
from types import TracebackType
if PY2:
# figure out size of _Py_ssize_t for Python 2:
if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'):
_Py_ssize_t = ctypes.c_int64
else:
_Py_ssize_t = ctypes.c_int
else:
# platform ssize_t on Python 3
_Py_ssize_t = ctypes.c_ssize_t
# regular python
class _PyObject(ctypes.Structure):
pass
_PyObject._fields_ = [
('ob_refcnt', _Py_ssize_t),
('ob_type', ctypes.POINTER(_PyObject))
]
# python with trace
if hasattr(sys, 'getobjects'):
class _PyObject(ctypes.Structure):
pass
_PyObject._fields_ = [
('_ob_next', ctypes.POINTER(_PyObject)),
('_ob_prev', ctypes.POINTER(_PyObject)),
('ob_refcnt', _Py_ssize_t),
('ob_type', ctypes.POINTER(_PyObject))
]
class _Traceback(_PyObject):
pass
_Traceback._fields_ = [
('tb_next', ctypes.POINTER(_Traceback)),
('tb_frame', ctypes.POINTER(_PyObject)),
('tb_lasti', ctypes.c_int),
('tb_lineno', ctypes.c_int)
]
def tb_set_next(tb, next):
"""Set the tb_next attribute of a traceback object."""
if not (isinstance(tb, TracebackType) and
(next is None or isinstance(next, TracebackType))):
raise TypeError('tb_set_next arguments must be traceback objects')
obj = _Traceback.from_address(id(tb))
if tb.tb_next is not None:
old = _Traceback.from_address(id(tb.tb_next))
old.ob_refcnt -= 1
if next is None:
obj.tb_next = ctypes.POINTER(_Traceback)()
else:
next = _Traceback.from_address(id(next))
next.ob_refcnt += 1
obj.tb_next = ctypes.pointer(next)
return tb_set_next
# try to get a tb_set_next implementation if we don't have transparent
# proxies.
tb_set_next = None
if tproxy is None:
# traceback.tb_next can be modified since CPython 3.7
if sys.version_info >= (3, 7):
def tb_set_next(tb, next):
tb.tb_next = next
else:
# On Python 3.6 and older, use ctypes
try:
tb_set_next = _init_ugly_crap()
except Exception:
pass
del _init_ugly_crap
package/bin/da_ess_contentupdate/jinja2/defaults.py
-56
View File
@@ -1,56 +0,0 @@
# -*- coding: utf-8 -*-
"""
jinja2.defaults
~~~~~~~~~~~~~~~
Jinja default filters and tags.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
from jinja2._compat import range_type
from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner, Namespace
# defaults for the parser / lexer
BLOCK_START_STRING = '{%'
BLOCK_END_STRING = '%}'
VARIABLE_START_STRING = '{{'
VARIABLE_END_STRING = '}}'
COMMENT_START_STRING = '{#'
COMMENT_END_STRING = '#}'
LINE_STATEMENT_PREFIX = None
LINE_COMMENT_PREFIX = None
TRIM_BLOCKS = False
LSTRIP_BLOCKS = False
NEWLINE_SEQUENCE = '\n'
KEEP_TRAILING_NEWLINE = False
# default filters, tests and namespace
from jinja2.filters import FILTERS as DEFAULT_FILTERS
from jinja2.tests import TESTS as DEFAULT_TESTS
DEFAULT_NAMESPACE = {
'range': range_type,
'dict': dict,
'lipsum': generate_lorem_ipsum,
'cycler': Cycler,
'joiner': Joiner,
'namespace': Namespace
}
# default policies
DEFAULT_POLICIES = {
'compiler.ascii_str': True,
'urlize.rel': 'noopener',
'urlize.target': None,
'truncate.leeway': 5,
'json.dumps_function': None,
'json.dumps_kwargs': {'sort_keys': True},
'ext.i18n.trimmed': False,
}
# export all constants
__all__ = tuple(x for x in locals().keys() if x.isupper())
package/bin/da_ess_contentupdate/jinja2/environment.py
-1276
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More