diff --git a/dist/escu/bin/splunklib/__init__.py b/dist/escu/bin/splunklib/__init__.py
old mode 100755
new mode 100644
index dd448e0d3a..525dc8eeda
--- a/dist/escu/bin/splunklib/__init__.py
+++ b/dist/escu/bin/splunklib/__init__.py
@@ -14,6 +14,7 @@
"""Python library for Splunk."""
-__version_info__ = (1, 6, 2)
+from __future__ import absolute_import
+from splunklib.six.moves import map
+__version_info__ = (1, 6, 16)
__version__ = ".".join(map(str, __version_info__))
-
diff --git a/dist/escu/bin/splunklib/binding.py b/dist/escu/bin/splunklib/binding.py
old mode 100755
new mode 100644
index e184c3ade0..c3121fb863
--- a/dist/escu/bin/splunklib/binding.py
+++ b/dist/escu/bin/splunklib/binding.py
@@ -24,30 +24,32 @@ If you want a friendlier interface to the Splunk REST API, use the
:mod:`splunklib.client` module.
"""
-import httplib
+from __future__ import absolute_import
+
+import io
import logging
import socket
import ssl
-import urllib
-import io
import sys
-import Cookie
-
from base64 import b64encode
+from contextlib import contextmanager
from datetime import datetime
from functools import wraps
-from StringIO import StringIO
-
-from contextlib import contextmanager
-
+from io import BytesIO
from xml.etree.ElementTree import XML
-try:
- from xml.etree.ElementTree import ParseError
-except ImportError, e:
- from xml.parsers.expat import ExpatError as ParseError
+
+from splunklib import six
+from splunklib.six import StringIO
+from splunklib.six.moves import urllib
from .data import record
+try:
+ from xml.etree.ElementTree import ParseError
+except ImportError as e:
+ from xml.parsers.expat import ExpatError as ParseError
+
+
__all__ = [
"AuthenticationError",
"connect",
@@ -78,6 +80,7 @@ def _parse_cookies(cookie_str, dictionary):
then updates the the dictionary with any key-value pairs found.
**Example**::
+
dictionary = {}
_parse_cookies('my=value', dictionary)
# Now the following is True
@@ -88,7 +91,7 @@ def _parse_cookies(cookie_str, dictionary):
:param dictionary: A dictionary to update with any found key-value pairs.
:type dictionary: ``dict``
"""
- parsed_cookie = Cookie.SimpleCookie(cookie_str)
+ parsed_cookie = six.moves.http_cookies.SimpleCookie(cookie_str)
for cookie in parsed_cookie.values():
dictionary[cookie.key] = cookie.coded_value
@@ -168,12 +171,12 @@ class UrlEncoded(str):
elif skip_encode:
return str.__new__(self, val)
elif encode_slash:
- return str.__new__(self, urllib.quote_plus(val))
+ return str.__new__(self, urllib.parse.quote_plus(val))
else:
# When subclassing str, just call str's __new__ method
# with your class and the value you want to have in the
# new string.
- return str.__new__(self, urllib.quote(val))
+ return str.__new__(self, urllib.parse.quote(val))
def __add__(self, other):
"""self + other
@@ -184,7 +187,7 @@ class UrlEncoded(str):
if isinstance(other, UrlEncoded):
return UrlEncoded(str.__add__(self, other), skip_encode=True)
else:
- return UrlEncoded(str.__add__(self, urllib.quote(other)), skip_encode=True)
+ return UrlEncoded(str.__add__(self, urllib.parse.quote(other)), skip_encode=True)
def __radd__(self, other):
"""other + self
@@ -195,7 +198,7 @@ class UrlEncoded(str):
if isinstance(other, UrlEncoded):
return UrlEncoded(str.__radd__(self, other), skip_encode=True)
else:
- return UrlEncoded(str.__add__(urllib.quote(other), self), skip_encode=True)
+ return UrlEncoded(str.__add__(urllib.parse.quote(other), self), skip_encode=True)
def __mod__(self, fields):
"""Interpolation into ``UrlEncoded``s is disabled.
@@ -205,7 +208,7 @@ class UrlEncoded(str):
"""
raise TypeError("Cannot interpolate into a UrlEncoded object.")
def __repr__(self):
- return "UrlEncoded(%s)" % repr(urllib.unquote(str(self)))
+ return "UrlEncoded(%s)" % repr(urllib.parse.unquote(str(self)))
@contextmanager
def _handle_auth_error(msg):
@@ -429,6 +432,8 @@ class Context(object):
:type port: ``integer``
:param scheme: The scheme for accessing the service (the default is "https").
:type scheme: "https" or "http"
+ :param verify: Enable (True) or disable (False) SSL verrification for https connections.
+ :type verify: ``Boolean``
:param sharing: The sharing mode for the namespace (the default is "user").
:type sharing: "global", "system", "app", or "user"
:param owner: The owner context of the namespace (optional, the default is "None").
@@ -445,6 +450,10 @@ class Context(object):
:type username: ``string``
:param password: The password for the Splunk account.
:type password: ``string``
+ :param splunkToken: Splunk authentication token
+ :type splunkToken: ``string``
+ :param headers: List of extra HTTP headers to send (optional).
+ :type headers: ``list`` of 2-tuples.
:param handler: The HTTP request handler (optional).
:returns: A ``Context`` instance.
@@ -461,7 +470,8 @@ class Context(object):
c = binding.Context(cookie="splunkd_8089=...")
"""
def __init__(self, handler=None, **kwargs):
- self.http = HttpLib(handler)
+ self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"),
+ cert_file=kwargs.get("cert_file")) # Default to False for backward compat
self.token = kwargs.get("token", _NoAuthenticationToken)
if self.token is None: # In case someone explicitly passes token=None
self.token = _NoAuthenticationToken
@@ -473,10 +483,12 @@ class Context(object):
self.username = kwargs.get("username", "")
self.password = kwargs.get("password", "")
self.basic = kwargs.get("basic", False)
+ self.bearerToken = kwargs.get("splunkToken", "")
self.autologin = kwargs.get("autologin", False)
+ self.additional_headers = kwargs.get("headers", [])
# Store any cookies in the self.http._cookies dict
- if kwargs.has_key("cookie") and kwargs['cookie'] not in [None, _NoAuthenticationToken]:
+ if "cookie" in kwargs and kwargs['cookie'] not in [None, _NoAuthenticationToken]:
_parse_cookies(kwargs["cookie"], self.http._cookies)
def get_cookies(self):
@@ -508,9 +520,12 @@ class Context(object):
:returns: A list of 2-tuples containing key and value
"""
if self.has_cookies():
- return [("Cookie", _make_cookie_header(self.get_cookies().items()))]
+ return [("Cookie", _make_cookie_header(list(self.get_cookies().items())))]
elif self.basic and (self.username and self.password):
- token = 'Basic %s' % b64encode("%s:%s" % (self.username, self.password))
+ token = 'Basic %s' % b64encode(("%s:%s" % (self.username, self.password)).encode('utf-8')).decode('ascii')
+ return [("Authorization", token)]
+ elif self.bearerToken:
+ token = 'Bearer %s' % self.bearerToken
return [("Authorization", token)]
elif self.token is _NoAuthenticationToken:
return []
@@ -609,7 +624,7 @@ class Context(object):
@_authentication
@_log_duration
- def get(self, path_segment, owner=None, app=None, sharing=None, **query):
+ def get(self, path_segment, owner=None, app=None, headers=None, sharing=None, **query):
"""Performs a GET operation from the REST path segment with the given
namespace and query.
@@ -632,6 +647,8 @@ class Context(object):
:type owner: ``string``
:param app: The app context of the namespace (optional).
:type app: ``string``
+ :param headers: List of extra HTTP headers to send (optional).
+ :type headers: ``list`` of 2-tuples.
:param sharing: The sharing mode of the namespace (optional).
:type sharing: ``string``
:param query: All other keyword arguments, which are used as query
@@ -659,10 +676,14 @@ class Context(object):
c.logout()
c.get('apps/local') # raises AuthenticationError
"""
+ if headers is None:
+ headers = []
+
path = self.authority + self._abspath(path_segment, owner=owner,
app=app, sharing=sharing)
logging.debug("GET request to %s (body: %s)", path, repr(query))
- response = self.http.get(path, self._auth_headers, **query)
+ all_headers = headers + self.additional_headers + self._auth_headers
+ response = self.http.get(path, all_headers, **query)
return response
@_authentication
@@ -703,7 +724,12 @@ class Context(object):
:type headers: ``list`` of 2-tuples.
:param query: All other keyword arguments, which are used as query
parameters.
- :type query: ``string``
+ :param body: Parameters to be used in the post body. If specified,
+ any parameters in the query will be applied to the URL instead of
+ the body. If a dict is supplied, the key-value pairs will be form
+ encoded. If a string is supplied, the body will be passed through
+ in the request unchanged.
+ :type body: ``dict`` or ``str``
:return: The response from the server.
:rtype: ``dict`` with keys ``body``, ``headers``, ``reason``,
and ``status``
@@ -734,7 +760,7 @@ class Context(object):
path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing)
logging.debug("POST request to %s (body: %s)", path, repr(query))
- all_headers = headers + self._auth_headers
+ all_headers = headers + self.additional_headers + self._auth_headers
response = self.http.post(path, all_headers, **query)
return response
@@ -800,7 +826,7 @@ class Context(object):
path = self.authority \
+ self._abspath(path_segment, owner=owner,
app=app, sharing=sharing)
- all_headers = headers + self._auth_headers
+ all_headers = headers + self.additional_headers + self._auth_headers
logging.debug("%s request to %s (headers: %s, body: %s)",
method, path, str(all_headers), repr(body))
response = self.http.request(path,
@@ -848,12 +874,17 @@ class Context(object):
# as credentials were passed in.
return
+ if self.bearerToken:
+ # Bearer auth mode requested, so this method is a nop as long
+ # as authentication token was passed in.
+ return
# Only try to get a token and updated cookie if username & password are specified
try:
response = self.http.post(
self.authority + self._abspath("/services/auth/login"),
username=self.username,
password=self.password,
+ headers=self.additional_headers,
cookie="1") # In Splunk 6.2+, passing "cookie=1" will return the "set-cookie" header
body = response.body.read()
@@ -964,6 +995,8 @@ def connect(**kwargs):
:type username: ``string``
:param password: The password for the Splunk account.
:type password: ``string``
+ :param headers: List of extra HTTP headers to send (optional).
+ :type headers: ``list`` of 2-tuples.
:param autologin: When ``True``, automatically tries to log in again if the
session terminates.
:type autologin: ``Boolean``
@@ -1011,7 +1044,7 @@ class AuthenticationError(HTTPError):
def __init__(self, message, cause):
# Put the body back in the response so that HTTPError's constructor can
# read it again.
- cause._response.body = StringIO(cause.body)
+ cause._response.body = BytesIO(cause.body)
HTTPError.__init__(self, cause._response, message)
@@ -1042,22 +1075,23 @@ class AuthenticationError(HTTPError):
# 'foo=1&foo=2&foo=3'.
def _encode(**kwargs):
items = []
- for key, value in kwargs.iteritems():
+ for key, value in six.iteritems(kwargs):
if isinstance(value, list):
items.extend([(key, item) for item in value])
else:
items.append((key, value))
- return urllib.urlencode(items)
+ return urllib.parse.urlencode(items)
# Crack the given url into (scheme, host, port, path)
def _spliturl(url):
- scheme, opaque = urllib.splittype(url)
- netloc, path = urllib.splithost(opaque)
- host, port = urllib.splitport(netloc)
+ parsed_url = urllib.parse.urlparse(url)
+ host = parsed_url.hostname
+ port = parsed_url.port
+ path = '?'.join((parsed_url.path, parsed_url.query)) if parsed_url.query else parsed_url.path
# Strip brackets if its an IPv6 address
if host.startswith('[') and host.endswith(']'): host = host[1:-1]
if port is None: port = DEFAULT_PORT
- return scheme, host, port, path
+ return parsed_url.scheme, host, port, path
# Given an HTTP request handler, this wrapper objects provides a related
# family of convenience methods built using that handler.
@@ -1100,9 +1134,14 @@ class HttpLib(object):
The response dictionary is returned directly by ``HttpLib``'s methods with
no further processing. By default, ``HttpLib`` calls the :func:`handler` function
to get a handler function.
+
+ If using the default handler, SSL verification can be disabled by passing verify=False.
"""
- def __init__(self, custom_handler=None):
- self.handler = handler() if custom_handler is None else custom_handler
+ def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None):
+ if custom_handler is None:
+ self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file)
+ else:
+ self.handler = custom_handler
self._cookies = {}
def delete(self, url, headers=None, **kwargs):
@@ -1185,14 +1224,16 @@ class HttpLib(object):
# We only use application/x-www-form-urlencoded if there is no other
# Content-Type header present. This can happen in cases where we
# send requests as application/json, e.g. for KV Store.
- if len(filter(lambda x: x[0].lower() == "content-type", headers)) == 0:
+ if len([x for x in headers if x[0].lower() == "content-type"]) == 0:
headers.append(("Content-Type", "application/x-www-form-urlencoded"))
body = kwargs.pop('body')
+ if isinstance(body, dict):
+ body = _encode(**body).encode('utf-8')
if len(kwargs) > 0:
url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True)
else:
- body = _encode(**kwargs)
+ body = _encode(**kwargs).encode('utf-8')
message = {
'method': "POST",
'headers': headers,
@@ -1226,7 +1267,7 @@ class HttpLib(object):
# If response.headers is a dict, get the key-value pairs as 2-tuples
# this is the case when using urllib2
if isinstance(response.headers, dict):
- key_value_tuples = response.headers.items()
+ key_value_tuples = list(response.headers.items())
for key, value in key_value_tuples:
if key.lower() == "set-cookie":
_parse_cookies(value, self._cookies)
@@ -1248,7 +1289,7 @@ class ResponseReader(io.RawIOBase):
def __init__(self, response, connection=None):
self._response = response
self._connection = connection
- self._buffer = ''
+ self._buffer = b''
def __str__(self):
return self.read()
@@ -1256,7 +1297,7 @@ class ResponseReader(io.RawIOBase):
@property
def empty(self):
"""Indicates whether there is any more data in the response."""
- return self.peek(1) == ""
+ return self.peek(1) == b""
def peek(self, size):
"""Nondestructively retrieves a given number of characters.
@@ -1273,8 +1314,8 @@ class ResponseReader(io.RawIOBase):
def close(self):
"""Closes this response."""
- if _connection:
- _connection.close()
+ if self._connection:
+ self._connection.close()
self._response.close()
def read(self, size = None):
@@ -1286,7 +1327,7 @@ class ResponseReader(io.RawIOBase):
"""
r = self._buffer
- self._buffer = ''
+ self._buffer = b''
if size is not None:
size -= len(r)
r = r + self._response.read(size)
@@ -1310,7 +1351,7 @@ class ResponseReader(io.RawIOBase):
return bytes_read
-def handler(key_file=None, cert_file=None, timeout=None):
+def handler(key_file=None, cert_file=None, timeout=None, verify=False):
"""This class returns an instance of the default HTTP request handler using
the values you provide.
@@ -1320,21 +1361,22 @@ def handler(key_file=None, cert_file=None, timeout=None):
:type cert_file: ``string``
:param `timeout`: The request time-out period, in seconds (optional).
:type timeout: ``integer`` or "None"
+ :param `verify`: Set to False to disable SSL verification on https connections.
+ :type verify: ``Boolean``
"""
def connect(scheme, host, port):
kwargs = {}
if timeout is not None: kwargs['timeout'] = timeout
if scheme == "http":
- return httplib.HTTPConnection(host, port, **kwargs)
+ return six.moves.http_client.HTTPConnection(host, port, **kwargs)
if scheme == "https":
if key_file is not None: kwargs['key_file'] = key_file
if cert_file is not None: kwargs['cert_file'] = cert_file
- # If running Python 2.7.9+, disable SSL certificate validation
- if sys.version_info >= (2,7,9) and key_file is None and cert_file is None:
+ if not verify:
kwargs['context'] = ssl._create_unverified_context()
- return httplib.HTTPSConnection(host, port, **kwargs)
+ return six.moves.http_client.HTTPSConnection(host, port, **kwargs)
raise ValueError("unsupported scheme: %s" % scheme)
def request(url, message, **kwargs):
@@ -1343,7 +1385,7 @@ def handler(key_file=None, cert_file=None, timeout=None):
head = {
"Content-Length": str(len(body)),
"Host": host,
- "User-Agent": "splunk-sdk-python/1.6.2",
+ "User-Agent": "splunk-sdk-python/1.6.16",
"Accept": "*/*",
"Connection": "Close",
} # defaults
diff --git a/dist/escu/bin/splunklib/client.py b/dist/escu/bin/splunklib/client.py
old mode 100755
new mode 100644
index d618e8a406..39b1dcc347
--- a/dist/escu/bin/splunklib/client.py
+++ b/dist/escu/bin/splunklib/client.py
@@ -58,18 +58,22 @@ attributes, and methods that are specific to each kind of entity. For example::
my_app.package() # Creates a compressed package of this application
"""
+import contextlib
import datetime
import json
-import urllib
import logging
-from time import sleep
-from datetime import datetime, timedelta
import socket
-import contextlib
+from datetime import datetime, timedelta
+from time import sleep
+
+from splunklib import six
+from splunklib.six.moves import urllib
-from .binding import Context, HTTPError, AuthenticationError, namespace, UrlEncoded, _encode, _make_cookie_header, _NoAuthenticationToken
-from .data import record
from . import data
+from .binding import (AuthenticationError, Context, HTTPError, UrlEncoded,
+ _encode, _make_cookie_header, _NoAuthenticationToken,
+ namespace)
+from .data import record
__all__ = [
"connect",
@@ -182,7 +186,7 @@ def _trailing(template, *targets):
def _filter_content(content, *args):
if len(args) > 0:
return record((k, content[k]) for k in args)
- return record((k, v) for k, v in content.iteritems()
+ return record((k, v) for k, v in six.iteritems(content)
if k not in ['eai:acl', 'eai:attributes', 'type'])
# Construct a resource path from the given base path + resource name
@@ -192,8 +196,11 @@ def _path(base, name):
# Load an atom record from the body of the given response
+# this will ultimately be sent to an xml ElementTree so we
+# should use the xmlcharrefreplace option
def _load_atom(response, match=None):
- return data.load(response.body.read(), match)
+ return data.load(response.body.read()
+ .decode('utf-8', 'xmlcharrefreplace'), match)
# Load an array of atom entries from the body of the given response
@@ -236,7 +243,7 @@ def _parse_atom_entry(entry):
metadata = _parse_atom_metadata(content)
# Filter some of the noise out of the content record
- content = record((k, v) for k, v in content.iteritems()
+ content = record((k, v) for k, v in six.iteritems(content)
if k not in ['eai:acl', 'eai:attributes'])
if 'type' in content:
@@ -288,6 +295,9 @@ def connect(**kwargs):
:type port: ``integer``
:param scheme: The scheme for accessing the service (the default is "https").
:type scheme: "https" or "http"
+ :param verify: Enable (True) or disable (False) SSL verrification for
+ https connections. (optional, the default is True)
+ :type verify: ``Boolean``
:param `owner`: The owner context of the namespace (optional).
:type owner: ``string``
:param `app`: The app context of the namespace (optional).
@@ -355,6 +365,9 @@ class Service(_BaseService):
:type port: ``integer``
:param scheme: The scheme for accessing the service (the default is "https").
:type scheme: "https" or "http"
+ :param verify: Enable (True) or disable (False) SSL verrification for
+ https connections. (optional, the default is True)
+ :type verify: ``Boolean``
:param `owner`: The owner context of the namespace (optional; use "-" for wildcard).
:type owner: ``string``
:param `app`: The app context of the namespace (optional; use "-" for wildcard).
@@ -450,6 +463,13 @@ class Service(_BaseService):
response = self.get("/services/server/info")
return _filter_content(_load_atom(response, MATCH_ENTRY_CONTENT))
+ def input(self, path, kind=None):
+ """Retrieves an input by path, and optionally kind.
+
+ :return: A :class:`Input` object.
+ """
+ return Input(self, path, kind=kind).refresh()
+
@property
def inputs(self):
"""Returns the collection of inputs configured on this Splunk instance.
@@ -559,9 +579,9 @@ class Service(_BaseService):
self.login()
if not self.restart_required:
return result
- except Exception, e:
+ except Exception as e:
sleep(1)
- raise Exception, "Operation time out."
+ raise Exception("Operation time out.")
@property
def restart_required(self):
@@ -884,7 +904,7 @@ class Entity(Endpoint):
try:
self[item]
return True
- except KeyError, AttributeError:
+ except (KeyError, AttributeError):
return False
def __eq__(self, other):
@@ -1081,7 +1101,7 @@ class Entity(Endpoint):
# text to be dispatched via HTTP. However, these links are already
# URL encoded when they arrive, and we need to mark them as such.
unquoted_links = dict([(k, UrlEncoded(v, skip_encode=True))
- for k,v in results['links'].iteritems()])
+ for k,v in six.iteritems(results['links'])])
results['links'] = unquoted_links
return results
@@ -1187,7 +1207,7 @@ class ReadOnlyCollection(Endpoint):
:raises ValueError: Raised if no namespace is specified and *key*
does not refer to a unique name.
- *Example*::
+ **Example**::
s = client.connect(...)
saved_searches = s.saved_searches
@@ -1290,7 +1310,7 @@ class ReadOnlyCollection(Endpoint):
# This has been factored out so that it can be easily
# overloaded by Configurations, which has to switch its
# entities' endpoints from its own properties/ to configs/.
- raw_path = urllib.unquote(state.links.alternate)
+ raw_path = urllib.parse.unquote(state.links.alternate)
if 'servicesNS/' in raw_path:
return _trailing(raw_path, 'servicesNS/', '/', '/')
elif 'services/' in raw_path:
@@ -1534,7 +1554,7 @@ class Collection(ReadOnlyCollection):
applications = s.apps
new_app = applications.create("my_fake_app")
"""
- if not isinstance(name, basestring):
+ if not isinstance(name, six.string_types):
raise InvalidNameException("%s is not a valid name for an entity." % name)
if 'namespace' in params:
namespace = params.pop('namespace')
@@ -1623,9 +1643,9 @@ class Collection(ReadOnlyCollection):
:rtype: ``dict`` with keys ``body``, ``headers``, ``reason``,
and ``status``
- Example:
+ **Example**::
- import splunklib.client
+ import splunklib.client
s = client.service(...)
saved_searches = s.saved_searches
saved_searches.get("my/saved/search") == \\
@@ -1717,7 +1737,7 @@ class Configurations(Collection):
# This has to be overridden to handle the plumbing of creating
# a ConfigurationFile (which is a Collection) instead of some
# Entity.
- if not isinstance(name, basestring):
+ if not isinstance(name, six.string_types):
raise ValueError("Invalid name: %s" % repr(name))
response = self.post(__conf=name)
if response.status == 303:
@@ -1811,7 +1831,7 @@ class StoragePasswords(Collection):
:return: The :class:`StoragePassword` object created.
"""
- if not isinstance(username, basestring):
+ if not isinstance(username, six.string_types):
raise ValueError("Invalid name: %s" % repr(username))
if realm is None:
@@ -1852,7 +1872,7 @@ class StoragePasswords(Collection):
name = UrlEncoded(realm, encode_slash=True) + ":" + UrlEncoded(username, encode_slash=True)
# Append the : expected at the end of the name
- if name[-1] is not ":":
+ if name[-1] != ":":
name = name + ":"
return Collection.delete(self, name)
@@ -1935,7 +1955,7 @@ class Index(Entity):
if host is not None: args['host'] = host
if source is not None: args['source'] = source
if sourcetype is not None: args['sourcetype'] = sourcetype
- path = UrlEncoded(PATH_RECEIVERS_STREAM + "?" + urllib.urlencode(args), skip_encode=True)
+ path = UrlEncoded(PATH_RECEIVERS_STREAM + "?" + urllib.parse.urlencode(args), skip_encode=True)
cookie_or_auth_header = "Authorization: Splunk %s\r\n" % \
(self.service.token if self.service.token is _NoAuthenticationToken
@@ -1949,12 +1969,12 @@ class Index(Entity):
# the connection open and use the Splunk extension headers to note
# the input mode
sock = self.service.connect()
- headers = ["POST %s HTTP/1.1\r\n" % self.service._abspath(path),
- "Host: %s:%s\r\n" % (self.service.host, int(self.service.port)),
- "Accept-Encoding: identity\r\n",
- cookie_or_auth_header,
- "X-Splunk-Input-Mode: Streaming\r\n",
- "\r\n"]
+ headers = [("POST %s HTTP/1.1\r\n" % str(self.service._abspath(path))).encode('utf-8'),
+ ("Host: %s:%s\r\n" % (self.service.host, int(self.service.port))).encode('utf-8'),
+ b"Accept-Encoding: identity\r\n",
+ cookie_or_auth_header.encode('utf-8'),
+ b"X-Splunk-Input-Mode: Streaming\r\n",
+ b"\r\n"]
for h in headers:
sock.write(h)
@@ -2028,8 +2048,7 @@ class Index(Entity):
self.refresh()
if self.content.totalEventCount != '0':
- raise OperationError, "Cleaning index %s took longer than %s seconds; timing out." %\
- (self.name, timeout)
+ raise OperationError("Cleaning index %s took longer than %s seconds; timing out." % (self.name, timeout))
finally:
# Restore original values
self.update(maxTotalDataSizeMB=tds, frozenTimePeriodInSecs=ftp)
@@ -2297,7 +2316,7 @@ class Inputs(Collection):
path = _path(
self.path + kindpath,
'%s:%s' % (kwargs['restrictToHost'], name) \
- if kwargs.has_key('restrictToHost') else name
+ if 'restrictToHost' in kwargs else name
)
return Input(self.service, path, kind)
@@ -2504,7 +2523,7 @@ class Inputs(Collection):
try:
path = UrlEncoded(path, skip_encode=True)
response = self.get(path, **kwargs)
- except HTTPError, he:
+ except HTTPError as he:
if he.status == 404: # No inputs of this kind
return []
entities = []
@@ -2516,7 +2535,7 @@ class Inputs(Collection):
# Unquote the URL, since all URL encoded in the SDK
# should be of type UrlEncoded, and all str should not
# be URL encoded.
- path = urllib.unquote(state.links.alternate)
+ path = urllib.parse.unquote(state.links.alternate)
entity = Input(self.service, path, kind, state=state)
entities.append(entity)
return entities
@@ -2542,7 +2561,7 @@ class Inputs(Collection):
# Unquote the URL, since all URL encoded in the SDK
# should be of type UrlEncoded, and all str should not
# be URL encoded.
- path = urllib.unquote(state.links.alternate)
+ path = urllib.parse.unquote(state.links.alternate)
entity = Input(self.service, path, kind, state=state)
entities.append(entity)
if 'offset' in kwargs:
@@ -3359,7 +3378,7 @@ class Users(Collection):
boris = users.create("boris", "securepassword", roles="user")
hilda = users.create("hilda", "anotherpassword", roles=["user","power"])
"""
- if not isinstance(username, basestring):
+ if not isinstance(username, six.string_types):
raise ValueError("Invalid username: %s" % str(username))
username = username.lower()
self.post(name=username, password=password, roles=roles, **params)
@@ -3370,7 +3389,7 @@ class Users(Collection):
state = _parse_atom_entry(entry)
entity = self.item(
self.service,
- urllib.unquote(state.links.alternate),
+ urllib.parse.unquote(state.links.alternate),
state=state)
return entity
@@ -3482,7 +3501,7 @@ class Roles(Collection):
roles = c.roles
paltry = roles.create("paltry", imported_roles="user", defaultApp="search")
"""
- if not isinstance(name, basestring):
+ if not isinstance(name, six.string_types):
raise ValueError("Invalid role name: %s" % str(name))
name = name.lower()
self.post(name=name, **params)
@@ -3493,7 +3512,7 @@ class Roles(Collection):
state = _parse_atom_entry(entry)
entity = self.item(
self.service,
- urllib.unquote(state.links.alternate),
+ urllib.parse.unquote(state.links.alternate),
state=state)
return entity
@@ -3544,11 +3563,11 @@ class KVStoreCollections(Collection):
:return: Result of POST request
"""
- for k, v in indexes.iteritems():
+ for k, v in six.iteritems(indexes):
if isinstance(v, dict):
v = json.dumps(v)
kwargs['index.' + k] = v
- for k, v in fields.iteritems():
+ for k, v in six.iteritems(fields):
kwargs['field.' + k] = v
return self.post(name=name, **kwargs)
@@ -3557,7 +3576,7 @@ class KVStoreCollection(Entity):
def data(self):
"""Returns data object for this Collection.
- :rtype: :class:`KVStoreData`
+ :rtype: :class:`KVStoreCollectionData`
"""
return KVStoreCollectionData(self)
@@ -3621,7 +3640,7 @@ class KVStoreCollectionData(object):
:return: Array of documents retrieved by query.
:rtype: ``array``
"""
- return json.loads(self._get('', **query).body.read())
+ return json.loads(self._get('', **query).body.read().decode('utf-8'))
def query_by_id(self, id):
"""
@@ -3633,7 +3652,7 @@ class KVStoreCollectionData(object):
:return: Document with id
:rtype: ``dict``
"""
- return json.loads(self._get(UrlEncoded(str(id))).body.read())
+ return json.loads(self._get(UrlEncoded(str(id))).body.read().decode('utf-8'))
def insert(self, data):
"""
@@ -3645,7 +3664,7 @@ class KVStoreCollectionData(object):
:return: _id of inserted object
:rtype: ``dict``
"""
- return json.loads(self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read())
+ return json.loads(self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))
def delete(self, query=None):
"""
@@ -3681,7 +3700,7 @@ class KVStoreCollectionData(object):
:return: id of replaced document
:rtype: ``dict``
"""
- return json.loads(self._post(UrlEncoded(str(id)), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read())
+ return json.loads(self._post(UrlEncoded(str(id)), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))
def batch_find(self, *dbqueries):
"""
@@ -3698,7 +3717,7 @@ class KVStoreCollectionData(object):
data = json.dumps(dbqueries)
- return json.loads(self._post('batch_find', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read())
+ return json.loads(self._post('batch_find', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))
def batch_save(self, *documents):
"""
@@ -3715,4 +3734,4 @@ class KVStoreCollectionData(object):
data = json.dumps(documents)
- return json.loads(self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read())
+ return json.loads(self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))
diff --git a/dist/escu/bin/splunklib/data.py b/dist/escu/bin/splunklib/data.py
old mode 100755
new mode 100644
index 54e0e7eb1c..dedbb33105
--- a/dist/escu/bin/splunklib/data.py
+++ b/dist/escu/bin/splunklib/data.py
@@ -12,11 +12,14 @@
# License for the specific language governing permissions and limitations
# under the License.
-"""The **splunklib.data** module reads the responses from splunkd in Atom Feed
+"""The **splunklib.data** module reads the responses from splunkd in Atom Feed
format, which is the format used by most of the REST API.
"""
+from __future__ import absolute_import
+import sys
from xml.etree.ElementTree import XML
+from splunklib import six
__all__ = ["load"]
@@ -56,10 +59,10 @@ def localname(xname):
return xname if rcurly == -1 else xname[rcurly+1:]
def load(text, match=None):
- """This function reads a string that contains the XML of an Atom Feed, then
- returns the
- data in a native Python structure (a ``dict`` or ``list``). If you also
- provide a tag name or path to match, only the matching sub-elements are
+ """This function reads a string that contains the XML of an Atom Feed, then
+ returns the
+ data in a native Python structure (a ``dict`` or ``list``). If you also
+ provide a tag name or path to match, only the matching sub-elements are
loaded.
:param text: The XML text to load.
@@ -74,12 +77,17 @@ def load(text, match=None):
'namespaces': [],
'names': {}
}
+
+ # Convert to unicode encoding in only python 2 for xml parser
+ if(sys.version_info < (3, 0, 0) and isinstance(text, unicode)):
+ text = text.encode('utf-8')
+
root = XML(text)
items = [root] if match is None else root.findall(match)
count = len(items)
- if count == 0:
+ if count == 0:
return None
- elif count == 1:
+ elif count == 1:
return load_root(items[0], nametable)
else:
return [load_root(item, nametable) for item in items]
@@ -88,7 +96,7 @@ def load(text, match=None):
def load_attrs(element):
if not hasattrs(element): return None
attrs = record()
- for key, value in element.attrib.iteritems():
+ for key, value in six.iteritems(element.attrib):
attrs[key] = value
return attrs
@@ -110,12 +118,12 @@ def load_elem(element, nametable=None):
if attrs is None: return name, value
if value is None: return name, attrs
# If value is simple, merge into attrs dict using special key
- if isinstance(value, str):
+ if isinstance(value, six.string_types):
attrs["$text"] = value
return name, attrs
# Both attrs & value are complex, so merge the two dicts, resolving collisions.
collision_keys = []
- for key, val in attrs.iteritems():
+ for key, val in six.iteritems(attrs):
if key in value and key in collision_keys:
value[key].append(val)
elif key in value and key not in collision_keys:
@@ -151,10 +159,10 @@ def load_value(element, nametable=None):
# No children, assume a simple text value
if count == 0:
text = element.text
- if text is None:
+ if text is None:
return None
text = text.strip()
- if len(text) == 0:
+ if len(text) == 0:
return None
return text
@@ -169,9 +177,9 @@ def load_value(element, nametable=None):
for child in children:
name, item = load_elem(child, nametable)
# If we have seen this name before, promote the value to a list
- if value.has_key(name):
+ if name in value:
current = value[name]
- if not isinstance(current, list):
+ if not isinstance(current, list):
value[name] = [current]
value[name].append(item)
else:
@@ -181,21 +189,21 @@ def load_value(element, nametable=None):
# A generic utility that enables "dot" access to dicts
class Record(dict):
- """This generic utility class enables dot access to members of a Python
+ """This generic utility class enables dot access to members of a Python
dictionary.
- Any key that is also a valid Python identifier can be retrieved as a field.
- So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to
- ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be
- retrieved as a field, because ``-`` and ``.`` are not allowed in
+ Any key that is also a valid Python identifier can be retrieved as a field.
+ So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to
+ ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be
+ retrieved as a field, because ``-`` and ``.`` are not allowed in
identifiers.
- Keys of the form ``a.b.c`` are very natural to write in Python as fields. If
- a group of keys shares a prefix ending in ``.``, you can retrieve keys as a
+ Keys of the form ``a.b.c`` are very natural to write in Python as fields. If
+ a group of keys shares a prefix ending in ``.``, you can retrieve keys as a
nested dictionary by calling only the prefix. For example, if ``r`` contains
keys ``'foo'``, ``'bar.baz'``, and ``'bar.qux'``, ``r.bar`` returns a record
- with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each
- one is placed into a nested dictionary, so you can write ``r.bar.qux`` or
+ with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each
+ one is placed into a nested dictionary, so you can write ``r.bar.qux`` or
``r['bar.qux']`` interchangeably.
"""
sep = '.'
@@ -207,7 +215,7 @@ class Record(dict):
def __getattr__(self, name):
try:
return self[name]
- except KeyError:
+ except KeyError:
raise AttributeError(name)
def __delattr__(self, name):
@@ -227,7 +235,7 @@ class Record(dict):
return dict.__getitem__(self, key)
key += self.sep
result = record()
- for k,v in self.iteritems():
+ for k,v in six.iteritems(self):
if not k.startswith(key):
continue
suffix = k[len(key):]
@@ -244,12 +252,12 @@ class Record(dict):
if len(result) == 0:
raise KeyError("No key or prefix: %s" % key)
return result
+
-
-def record(value=None):
- """This function returns a :class:`Record` instance constructed with an
+def record(value=None):
+ """This function returns a :class:`Record` instance constructed with an
initial value that you provide.
-
+
:param `value`: An initial record value.
:type `value`: ``dict``
"""
diff --git a/dist/escu/bin/splunklib/modularinput/__init__.py b/dist/escu/bin/splunklib/modularinput/__init__.py
old mode 100755
new mode 100644
diff --git a/dist/escu/bin/splunklib/modularinput/argument.py b/dist/escu/bin/splunklib/modularinput/argument.py
old mode 100755
new mode 100644
index 5eb3f1259e..04214d16d9
--- a/dist/escu/bin/splunklib/modularinput/argument.py
+++ b/dist/escu/bin/splunklib/modularinput/argument.py
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import
try:
import xml.etree.ElementTree as ET
except ImportError:
@@ -20,7 +21,7 @@ except ImportError:
class Argument(object):
"""Class representing an argument to a modular input kind.
- ``Argument`` is meant to be used with ``Scheme`` to generate an XML
+ ``Argument`` is meant to be used with ``Scheme`` to generate an XML
definition of the modular input kind that Splunk understands.
``name`` is the only required parameter for the constructor.
@@ -53,9 +54,9 @@ class Argument(object):
:param name: ``string``, identifier for this argument in Splunk.
:param description: ``string``, human-readable description of the argument.
:param validation: ``string`` specifying how the argument should be validated, if using internal validation.
- If using external validation, this will be ignored.
+ If using external validation, this will be ignored.
:param data_type: ``string``, data type of this field; use the class constants.
- "data_type_boolean", "data_type_number", or "data_type_string".
+ "data_type_boolean", "data_type_number", or "data_type_string".
:param required_on_edit: ``Boolean``, whether this arg is required when editing an existing modular input of this kind.
:param required_on_create: ``Boolean``, whether this arg is required when creating a modular input of this kind.
:param title: ``String``, a human-readable title for the argument.
diff --git a/dist/escu/bin/splunklib/modularinput/event.py b/dist/escu/bin/splunklib/modularinput/event.py
old mode 100755
new mode 100644
index de1d4f19e6..9cd6cf3ae8
--- a/dist/escu/bin/splunklib/modularinput/event.py
+++ b/dist/escu/bin/splunklib/modularinput/event.py
@@ -12,6 +12,10 @@
# License for the specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import
+from io import TextIOBase
+from splunklib.six import ensure_text
+
try:
import xml.etree.cElementTree as ET
except ImportError as ie:
@@ -103,5 +107,8 @@ class Event(object):
if self.done:
ET.SubElement(event, "done")
- stream.write(ET.tostring(event))
+ if isinstance(stream, TextIOBase):
+ stream.write(ensure_text(ET.tostring(event)))
+ else:
+ stream.write(ET.tostring(event))
stream.flush()
\ No newline at end of file
diff --git a/dist/escu/bin/splunklib/modularinput/event_writer.py b/dist/escu/bin/splunklib/modularinput/event_writer.py
old mode 100755
new mode 100644
index 418405fceb..3e4321016e
--- a/dist/escu/bin/splunklib/modularinput/event_writer.py
+++ b/dist/escu/bin/splunklib/modularinput/event_writer.py
@@ -12,18 +12,20 @@
# License for the specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import
import sys
+from io import TextIOWrapper, TextIOBase
+from splunklib.six import ensure_str
from .event import ET
try:
- from cStringIO import StringIO
+ from splunklib.six.moves import cStringIO as StringIO
except ImportError:
- from StringIO import StringIO
+ from splunklib.six import StringIO
class EventWriter(object):
"""``EventWriter`` writes events and error messages to Splunk from a modular input.
-
Its two important methods are ``writeEvent``, which takes an ``Event`` object,
and ``log``, which takes a severity and an error message.
"""
@@ -76,9 +78,10 @@ class EventWriter(object):
:param document: An ``ElementTree`` object.
"""
- self._out.write(ET.tostring(document))
+ self._out.write(ensure_str(ET.tostring(document)))
self._out.flush()
def close(self):
"""Write the closing tag to make this XML well formed."""
- self._out.write("")
\ No newline at end of file
+ self._out.write("")
+ self._out.flush()
diff --git a/dist/escu/bin/splunklib/modularinput/input_definition.py b/dist/escu/bin/splunklib/modularinput/input_definition.py
old mode 100755
new mode 100644
index 3a2e1faefa..fdc7cbb3f3
--- a/dist/escu/bin/splunklib/modularinput/input_definition.py
+++ b/dist/escu/bin/splunklib/modularinput/input_definition.py
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import
try:
import xml.etree.cElementTree as ET
except ImportError as ie:
diff --git a/dist/escu/bin/splunklib/modularinput/scheme.py b/dist/escu/bin/splunklib/modularinput/scheme.py
old mode 100755
new mode 100644
index c3aa8120ed..4104e4a3f8
--- a/dist/escu/bin/splunklib/modularinput/scheme.py
+++ b/dist/escu/bin/splunklib/modularinput/scheme.py
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import
try:
import xml.etree.cElementTree as ET
except ImportError:
@@ -54,7 +55,7 @@ class Scheme(object):
def to_xml(self):
"""Creates an ``ET.Element`` representing self, then returns it.
- :returns root, an ``ET.Element`` representing this scheme.
+ :returns: an ``ET.Element`` representing this scheme.
"""
root = ET.Element("scheme")
diff --git a/dist/escu/bin/splunklib/modularinput/script.py b/dist/escu/bin/splunklib/modularinput/script.py
old mode 100755
new mode 100644
index dddca8ad79..8595dc4bdb
--- a/dist/escu/bin/splunklib/modularinput/script.py
+++ b/dist/escu/bin/splunklib/modularinput/script.py
@@ -12,14 +12,16 @@
# License for the specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import
from abc import ABCMeta, abstractmethod
-from urlparse import urlsplit
+from splunklib.six.moves.urllib.parse import urlsplit
import sys
from ..client import Service
from .event_writer import EventWriter
from .input_definition import InputDefinition
from .validation_definition import ValidationDefinition
+from splunklib import six
try:
import xml.etree.cElementTree as ET
@@ -27,7 +29,7 @@ except ImportError:
import xml.etree.ElementTree as ET
-class Script(object):
+class Script(six.with_metaclass(ABCMeta, object)):
"""An abstract base class for implementing modular inputs.
Subclasses should override ``get_scheme``, ``stream_events``,
@@ -37,7 +39,6 @@ class Script(object):
The ``run`` function is used to run modular inputs; it typically should
not be overridden.
"""
- __metaclass__ = ABCMeta
def __init__(self):
self._input_definition = None
@@ -101,10 +102,10 @@ class Script(object):
err_string = "ERROR Invalid arguments to modular input script:" + ' '.join(
args)
event_writer._err.write(err_string)
+ return 1
except Exception as e:
- err_string = EventWriter.ERROR + str(e.message)
- event_writer._err.write(err_string)
+ event_writer.log(EventWriter.ERROR, str(e))
return 1
@property
@@ -116,9 +117,9 @@ class Script(object):
available as soon as the :code:`Script.stream_events` method is
called.
- :return: :class:splunklib.client.Service. A value of None is returned,
- if you call this method before the :code:`Script.stream_events` method
- is called.
+ :return: :class:`splunklib.client.Service`. A value of None is returned,
+ if you call this method before the :code:`Script.stream_events` method
+ is called.
"""
if self._service is not None:
diff --git a/dist/escu/bin/splunklib/modularinput/utils.py b/dist/escu/bin/splunklib/modularinput/utils.py
old mode 100755
new mode 100644
index f9de82f2bf..853694a0da
--- a/dist/escu/bin/splunklib/modularinput/utils.py
+++ b/dist/escu/bin/splunklib/modularinput/utils.py
@@ -14,6 +14,8 @@
# File for utility functions
+from __future__ import absolute_import
+from splunklib.six.moves import zip
def xml_compare(expected, found):
"""Checks equality of two ``ElementTree`` objects.
@@ -69,4 +71,4 @@ def parse_xml_data(parent_node, child_node_tag):
data[child.get("name")][param.get("name")] = parse_parameters(param)
elif "item" == parent_node.tag:
data[child.get("name")] = parse_parameters(child)
- return data
\ No newline at end of file
+ return data
diff --git a/dist/escu/bin/splunklib/modularinput/validation_definition.py b/dist/escu/bin/splunklib/modularinput/validation_definition.py
old mode 100755
new mode 100644
index 72f8e7bec2..3bbe9760e5
--- a/dist/escu/bin/splunklib/modularinput/validation_definition.py
+++ b/dist/escu/bin/splunklib/modularinput/validation_definition.py
@@ -13,6 +13,7 @@
# under the License.
+from __future__ import absolute_import
try:
import xml.etree.cElementTree as ET
except ImportError as ie:
@@ -27,7 +28,7 @@ class ValidationDefinition(object):
**Example**::
- ``v = ValidationDefinition()``
+ v = ValidationDefinition()
"""
def __init__(self):
@@ -45,23 +46,25 @@ class ValidationDefinition(object):
The XML typically will look like this:
- ````
- `` myHost``
- `` https://127.0.0.1:8089``
- `` 123102983109283019283``
- `` /opt/splunk/var/lib/splunk/modinputs``
- `` - ``
- `` value1``
- `` ``
- `` value2``
- `` value3``
- `` value4``
- `` ``
- ``
``
- ````
+ .. code-block:: xml
+
+
+ myHost
+ https://127.0.0.1:8089
+ 123102983109283019283
+ /opt/splunk/var/lib/splunk/modinputs
+ -
+ value1
+
+ value2
+ value3
+ value4
+
+
+
:param stream: ``Stream`` containing XML to parse.
- :return definition: A ``ValidationDefinition`` object.
+ :return: A ``ValidationDefinition`` object.
"""
diff --git a/dist/escu/bin/splunklib/ordereddict.py b/dist/escu/bin/splunklib/ordereddict.py
old mode 100755
new mode 100644
diff --git a/dist/escu/bin/splunklib/results.py b/dist/escu/bin/splunklib/results.py
old mode 100755
new mode 100644
index ffc9b0bee9..20501c5b74
--- a/dist/escu/bin/splunklib/results.py
+++ b/dist/escu/bin/splunklib/results.py
@@ -32,6 +32,11 @@ as follows:::
print "Results are a preview: %s" % reader.is_preview
"""
+from __future__ import absolute_import
+
+from io import BytesIO
+
+from splunklib import six
try:
import xml.etree.cElementTree as et
except:
@@ -43,9 +48,9 @@ except ImportError:
from .ordereddict import OrderedDict
try:
- from cStringIO import StringIO
+ from splunklib.six.moves import cStringIO as StringIO
except:
- from StringIO import StringIO
+ from splunklib.six import StringIO
__all__ = [
"ResultsReader",
@@ -95,13 +100,13 @@ class _ConcatenatedStream(object):
If *n* is ``None``, return all available characters.
"""
- response = ""
+ response = b""
while len(self.streams) > 0 and (n is None or n > 0):
txt = self.streams[0].read(n)
response += txt
if n is not None:
n -= len(txt)
- if n > 0 or n is None:
+ if n is None or n > 0:
del self.streams[0]
return response
@@ -126,17 +131,17 @@ class _XMLDTDFilter(object):
If *n* is ``None``, return all available characters.
"""
- response = ""
+ response = b""
while n is None or n > 0:
c = self.stream.read(1)
- if c == "":
+ if c == b"":
break
- elif c == "<":
+ elif c == b"<":
c += self.stream.read(1)
- if c == "":
+ if q == b">":
break
else:
response += c
@@ -194,7 +199,7 @@ class ResultsReader(object):
# we remove all the DTD definitions inline, then wrap the
# fragments in a fiction element to make the parser happy.
stream = _XMLDTDFilter(stream)
- stream = _ConcatenatedStream(StringIO(""), stream, StringIO(""))
+ stream = _ConcatenatedStream(BytesIO(b""), stream, BytesIO(b""))
self.is_preview = None
self._gen = self._parse_results(stream)
@@ -202,7 +207,9 @@ class ResultsReader(object):
return self
def next(self):
- return self._gen.next()
+ return next(self._gen)
+
+ __next__ = next
def _parse_results(self, stream):
"""Parse results and messages out of *stream*."""
@@ -233,7 +240,7 @@ class ResultsReader(object):
if event == 'start':
values = []
elif event == 'end':
- field_name = elem.attrib['k'].encode('utf8')
+ field_name = elem.attrib['k']
if len(values) == 1:
result[field_name] = values[0]
else:
@@ -254,7 +261,7 @@ class ResultsReader(object):
def __itertext(self):
tag = self.tag
- if not isinstance(tag, basestring) and tag is not None:
+ if not isinstance(tag, six.string_types) and tag is not None:
return
if self.text:
yield self.text
@@ -265,7 +272,7 @@ class ResultsReader(object):
yield e.tail
text = "".join(__itertext(elem))
- values.append(text.encode('utf8'))
+ values.append(text)
elem.clear()
elif elem.tag == 'msg':
@@ -273,7 +280,7 @@ class ResultsReader(object):
msg_type = elem.attrib['type']
elif event == 'end':
text = elem.text if elem.text is not None else ""
- yield Message(msg_type, text.encode('utf8'))
+ yield Message(msg_type, text)
elem.clear()
except SyntaxError as pe:
# This is here to handle the same incorrect return from
diff --git a/dist/escu/bin/splunklib/searchcommands/__init__.py b/dist/escu/bin/splunklib/searchcommands/__init__.py
old mode 100755
new mode 100644
index 12b14f32c7..c56c510d59
--- a/dist/escu/bin/splunklib/searchcommands/__init__.py
+++ b/dist/escu/bin/splunklib/searchcommands/__init__.py
@@ -30,7 +30,7 @@
field-name = ( "_" / alpha ) *( alpha / digit / "_" / "." / "-" )
It does not show that :code:`field-name` values may be comma-separated. This is because Splunk strips commas from
- the command line. A search command will never see them.
+ the command line. A search command will never see them.
2. Search commands targeting versions of Splunk prior to 6.3 must be statically configured as follows:
@@ -134,7 +134,7 @@
.. topic:: References
- 1. `Search command style guide `_
+ 1. `Search command style guide `__
2. `Commands.conf.spec `_
diff --git a/dist/escu/bin/splunklib/searchcommands/decorators.py b/dist/escu/bin/splunklib/searchcommands/decorators.py
old mode 100755
new mode 100644
index 1a0400fc5d..36590a76bb
--- a/dist/escu/bin/splunklib/searchcommands/decorators.py
+++ b/dist/escu/bin/splunklib/searchcommands/decorators.py
@@ -15,6 +15,7 @@
# under the License.
from __future__ import absolute_import, division, print_function, unicode_literals
+from splunklib import six
try:
from collections import OrderedDict # must be python 2.7
@@ -22,7 +23,7 @@ except ImportError:
from ..ordereddict import OrderedDict
from inspect import getmembers, isclass, isfunction
-from itertools import imap
+from splunklib.six.moves import map as imap
from .internals import ConfigurationSettingsType, json_encode_string
from .validators import OptionName
@@ -35,7 +36,7 @@ class Configuration(object):
variable to search command classes that don't have one. The :code:`name` is derived from the name of the class.
By convention command class names end with the word "Command". To derive :code:`name` the word "Command" is removed
from the end of the class name and then converted to lower case for conformance with the `Search command style guide
- `_
+ `__
"""
def __init__(self, o=None, **kwargs):
@@ -69,15 +70,15 @@ class Configuration(object):
# Set command name
name = o.__name__
- if name.endswith(b'Command'):
- name = name[:-len(b'Command')]
- o.name = unicode(name.lower())
+ if name.endswith('Command'):
+ name = name[:-len('Command')]
+ o.name = six.text_type(name.lower())
# Construct ConfigurationSettings instance for the command class
o.ConfigurationSettings = ConfigurationSettingsType(
- module=o.__module__ + b'.' + o.__name__,
- name=b'ConfigurationSettings',
+ module=o.__module__ + '.' + o.__name__,
+ name='ConfigurationSettings',
bases=(o.ConfigurationSettings,))
ConfigurationSetting.fix_up(o.ConfigurationSettings, self.settings)
@@ -138,7 +139,7 @@ class ConfigurationSetting(property):
for name, setting in definitions:
if setting._name is None:
- setting._name = name = unicode(name)
+ setting._name = name = six.text_type(name)
else:
name = setting._name
@@ -195,8 +196,8 @@ class ConfigurationSetting(property):
del values[name]
if len(values) > 0:
- settings = sorted(list(values.iteritems()))
- settings = imap(lambda (n, v): '{}={}'.format(n, repr(v)), settings)
+ settings = sorted(list(six.iteritems(values)))
+ settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])), settings)
raise AttributeError('Inapplicable configuration settings: ' + ', '.join(settings))
cls.configuration_setting_definitions = definitions
@@ -228,8 +229,9 @@ class Option(property):
Short form (recommended). When you are satisfied with built-in or custom validation behaviors.
- .. code-block:: python
+ .. code-block:: python
:linenos:
+
from splunklib.searchcommands.decorators import Option
from splunklib.searchcommands.validators import Fieldname
@@ -246,8 +248,9 @@ class Option(property):
also provide a deleter. You must be prepared to accept a value of :const:`None` which indicates that your
:code:`Option` is unset.
- .. code-block:: python
+ .. code-block:: python
:linenos:
+
from splunklib.searchcommands import Option
@Option()
@@ -357,7 +360,7 @@ class Option(property):
self._option = option
self._is_set = False
validator = self.validator
- self._format = unicode if validator is None else validator.format
+ self._format = six.text_type if validator is None else validator.format
def __repr__(self):
return '(' + repr(self.name) + ', ' + repr(self._format(self.value)) + ')'
@@ -417,24 +420,24 @@ class Option(property):
def __init__(self, command):
definitions = type(command).option_definitions
item_class = Option.Item
- OrderedDict.__init__(self, imap(lambda (name, option): (option.name, item_class(command, option)), definitions))
+ OrderedDict.__init__(self, ((option.name, item_class(command, option)) for (name, option) in definitions))
def __repr__(self):
- text = 'Option.View([' + ','.join(imap(lambda item: repr(item), self.itervalues())) + '])'
+ text = 'Option.View([' + ','.join(imap(lambda item: repr(item), six.itervalues(self))) + '])'
return text
def __str__(self):
- text = ' '.join([str(item) for item in self.itervalues() if item.is_set])
+ text = ' '.join([str(item) for item in six.itervalues(self) if item.is_set])
return text
# region Methods
def get_missing(self):
- missing = [item.name for item in self.itervalues() if item.is_required and not item.is_set]
+ missing = [item.name for item in six.itervalues(self) if item.is_required and not item.is_set]
return missing if len(missing) > 0 else None
def reset(self):
- for value in self.itervalues():
+ for value in six.itervalues(self):
value.reset()
pass
diff --git a/dist/escu/bin/splunklib/searchcommands/environment.py b/dist/escu/bin/splunklib/searchcommands/environment.py
old mode 100755
new mode 100644
index 785042c1b3..e92018f6a4
--- a/dist/escu/bin/splunklib/searchcommands/environment.py
+++ b/dist/escu/bin/splunklib/searchcommands/environment.py
@@ -18,7 +18,8 @@ from __future__ import absolute_import, division, print_function, unicode_litera
from logging import getLogger, root, StreamHandler
from logging.config import fileConfig
-from os import chdir, environ, getcwdu, path
+from os import chdir, environ, path
+from splunklib.six.moves import getcwd
import sys
@@ -96,7 +97,7 @@ def configure_logging(logger_name, filename=None):
filename = path.realpath(filename)
if filename != _current_logging_configuration_file:
- working_directory = getcwdu()
+ working_directory = getcwd()
chdir(app_root)
try:
fileConfig(filename, {'SPLUNK_HOME': splunk_home})
@@ -112,7 +113,7 @@ def configure_logging(logger_name, filename=None):
_current_logging_configuration_file = None
-splunk_home = path.abspath(path.join(getcwdu(), environ.get('SPLUNK_HOME', '')))
+splunk_home = path.abspath(path.join(getcwd(), environ.get('SPLUNK_HOME', '')))
app_file = getattr(sys.modules['__main__'], '__file__', sys.executable)
app_root = path.dirname(path.abspath(path.dirname(app_file)))
diff --git a/dist/escu/bin/splunklib/searchcommands/eventing_command.py b/dist/escu/bin/splunklib/searchcommands/eventing_command.py
old mode 100755
new mode 100644
index fde7aadbc7..27dc13a3af
--- a/dist/escu/bin/splunklib/searchcommands/eventing_command.py
+++ b/dist/escu/bin/splunklib/searchcommands/eventing_command.py
@@ -16,7 +16,8 @@
from __future__ import absolute_import, division, print_function, unicode_literals
-from itertools import imap
+from splunklib import six
+from splunklib.six.moves import map as imap
from .decorators import ConfigurationSetting
from .search_command import SearchCommand
@@ -113,10 +114,10 @@ class EventingCommand(SearchCommand):
''')
- type = ConfigurationSetting(readonly=True, value='eventing', doc='''
+ type = ConfigurationSetting(readonly=True, value='events', doc='''
Command type
- Fixed: :const:`'eventing'`.
+ Fixed: :const:`'events'`.
Supported by: SCP 2
@@ -135,8 +136,14 @@ class EventingCommand(SearchCommand):
raise AttributeError('No EventingCommand.transform override')
SearchCommand.ConfigurationSettings.fix_up(command)
+ # TODO: Stop looking like a dictionary because we don't obey the semantics
+ # N.B.: Does not use Python 2 dict copy semantics
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
- return imap(lambda (name, value): (name, 'events' if name == 'type' else value), iteritems)
+ return imap(lambda name_value: (name_value[0], 'events' if name_value[0] == 'type' else name_value[1]), iteritems)
+
+ # N.B.: Does not use Python 3 dict view semantics
+ if not six.PY2:
+ items = iteritems
# endregion
diff --git a/dist/escu/bin/splunklib/searchcommands/external_search_command.py b/dist/escu/bin/splunklib/searchcommands/external_search_command.py
old mode 100755
new mode 100644
index 4ad91a58cf..c2306241eb
--- a/dist/escu/bin/splunklib/searchcommands/external_search_command.py
+++ b/dist/escu/bin/splunklib/searchcommands/external_search_command.py
@@ -20,6 +20,7 @@ from logging import getLogger
import os
import sys
import traceback
+from splunklib import six
if sys.platform == 'win32':
from signal import signal, CTRL_BREAK_EVENT, SIGBREAK, SIGINT, SIGTERM
@@ -36,11 +37,11 @@ class ExternalSearchCommand(object):
"""
def __init__(self, path, argv=None, environ=None):
- if not isinstance(path, (bytes, unicode)):
+ if not isinstance(path, (bytes, six.text_type)):
raise ValueError('Expected a string value for path, not {}'.format(repr(path)))
self._logger = getLogger(self.__class__.__name__)
- self._path = unicode(path)
+ self._path = six.text_type(path)
self._argv = None
self._environ = None
@@ -89,7 +90,7 @@ class ExternalSearchCommand(object):
self._execute(self._path, self._argv, self._environ)
except:
error_type, error, tb = sys.exc_info()
- message = 'Command execution failed: ' + unicode(error)
+ message = 'Command execution failed: ' + six.text_type(error)
self._logger.error(message + '\nTraceback:\n' + ''.join(traceback.format_tb(tb)))
sys.exit(1)
@@ -104,13 +105,13 @@ class ExternalSearchCommand(object):
:param argv: Argument list.
:type argv: list or tuple
- The arguments to the child process should start with the name of the command being run, but this is not
- enforced. A value of :const:`None` specifies that the base name of path name :param:`path` should be used.
+ The arguments to the child process should start with the name of the command being run, but this is not
+ enforced. A value of :const:`None` specifies that the base name of path name :param:`path` should be used.
:param environ: A mapping which is used to define the environment variables for the new process.
:type environ: dict or None.
- This mapping is used instead of the current process’s environment. A value of :const:`None` specifies that
- the :data:`os.environ` mapping should be used.
+ This mapping is used instead of the current process’s environment. A value of :const:`None` specifies that
+ the :data:`os.environ` mapping should be used.
:return: None
diff --git a/dist/escu/bin/splunklib/searchcommands/generating_command.py b/dist/escu/bin/splunklib/searchcommands/generating_command.py
old mode 100755
new mode 100644
index 3bd0192318..724d45dd95
--- a/dist/escu/bin/splunklib/searchcommands/generating_command.py
+++ b/dist/escu/bin/splunklib/searchcommands/generating_command.py
@@ -19,7 +19,8 @@ from __future__ import absolute_import, division, print_function, unicode_litera
from .decorators import ConfigurationSetting
from .search_command import SearchCommand
-from itertools import imap, ifilter
+from splunklib import six
+from splunklib.six.moves import map as imap, filter as ifilter
# P1 [O] TODO: Discuss generates_timeorder in the class-level documentation for GeneratingCommand
@@ -56,7 +57,7 @@ class GeneratingCommand(SearchCommand):
+==========+=====================================+============================================+
| streams | streaming=True[,local=[True|False]] | type='streaming'[,distributed=[true|false] |
+----------+-------------------------------------+--------------------------------------------+
- | events | retainsevents=True, streaming=False | type='eventing' |
+ | events | retainsevents=True, streaming=False | type='events' |
+----------+-------------------------------------+--------------------------------------------+
| reports | streaming=False | type='reporting' |
+----------+-------------------------------------+--------------------------------------------+
@@ -92,9 +93,10 @@ class GeneratingCommand(SearchCommand):
+==========+===================================================+===================================================+
| streams | 1. Add this line to your command's stanza in | 1. Add this configuration setting to your code: |
| | | |
- | | default/commands.conf. | .. code-block:: python |
- | | .. code-block:: python | @Configuration(distributed=True) |
- | | local = false | class SomeCommand(GeneratingCommand) |
+ | | default/commands.conf:: | .. code-block:: python |
+ | | | |
+ | | local = false | @Configuration(distributed=True) |
+ | | | class SomeCommand(GeneratingCommand) |
| | | ... |
| | 2. Restart splunk | |
| | | 2. You are good to go; no need to restart Splunk |
@@ -112,29 +114,33 @@ class GeneratingCommand(SearchCommand):
| | settings to your command class: | setting to your command class: |
| | | |
| | .. code-block:: python | .. code-block:: python |
- | | @Configuration( | @Configuration(type='eventing') |
+ | | | |
+ | | @Configuration( | @Configuration(type='events') |
| | retainsevents=True, streaming=False) | class SomeCommand(GeneratingCommand) |
| | class SomeCommand(GeneratingCommand) | ... |
| | ... | |
| | | |
| | Or add these lines to default/commands.conf: | |
| | | |
- | | .. code-block:: | |
- | | retains events = true | |
+ | | .. code-block:: text | |
+ | | | |
+ | | retainsevents = true | |
| | streaming = false | |
+----------+---------------------------------------------------+---------------------------------------------------+
Configure your command class like this, if you wish to support both protocols:
- .. code-block:: python
- @Configuration(type='eventing', retainsevents=True, streaming=False)
+ .. code-block:: python
+
+ @Configuration(type='events', retainsevents=True, streaming=False)
class SomeCommand(GeneratingCommand)
...
You might also consider adding these lines to commands.conf instead of adding them to your command class:
- .. code-block:: python
- retains events = false
+ .. code-block:: python
+
+ retainsevents = false
streaming = false
Reporting Generating command
@@ -149,28 +155,32 @@ class GeneratingCommand(SearchCommand):
| | settings to your command class: | setting to your command class: |
| | | |
| | .. code-block:: python | .. code-block:: python |
+ | | | |
| | @Configuration(retainsevents=False) | @Configuration(type='reporting') |
| | class SomeCommand(GeneratingCommand) | class SomeCommand(GeneratingCommand) |
| | ... | ... |
| | | |
| | Or add this lines to default/commands.conf: | |
| | | |
- | | .. code-block:: | |
- | | retains events = false | |
+ | | .. code-block:: text | |
+ | | | |
+ | | retainsevents = false | |
| | streaming = false | |
+----------+---------------------------------------------------+---------------------------------------------------+
Configure your command class like this, if you wish to support both protocols:
- .. code-block:: python
+ .. code-block:: python
+
@Configuration(type='reporting', streaming=False)
class SomeCommand(GeneratingCommand)
...
You might also consider adding these lines to commands.conf instead of adding them to your command class:
- .. code-block:: python
- retains events = false
+ .. code-block:: text
+
+ retainsevents = false
streaming = false
"""
@@ -193,9 +203,23 @@ class GeneratingCommand(SearchCommand):
:return: `None`.
"""
- self._record_writer.write_records(self.generate())
+ if self._protocol_version == 2:
+ self._execute_v2(ifile, self.generate())
+ else:
+ assert self._protocol_version == 1
+ self._record_writer.write_records(self.generate())
self.finish()
+ def _execute_chunk_v2(self, process, chunk):
+ count = 0
+ for row in process:
+ self._record_writer.write_record(row)
+ count += 1
+ if count == self._record_writer._maxresultrows:
+ self._finished = False
+ return
+ self._finished = True
+
# endregion
# region Types
@@ -280,7 +304,7 @@ class GeneratingCommand(SearchCommand):
==================== ======================================================================================
Value Description
-------------------- --------------------------------------------------------------------------------------
- :const:`'eventing'` Runs as the first command in the Splunk events pipeline. Cannot be distributed.
+ :const:`'events'` Runs as the first command in the Splunk events pipeline. Cannot be distributed.
:const:`'reporting'` Runs as the first command in the Splunk reports pipeline. Cannot be distributed.
:const:`'streaming'` Runs as the first command in the Splunk streams pipeline. May be distributed.
==================== ======================================================================================
@@ -303,16 +327,22 @@ class GeneratingCommand(SearchCommand):
if command.generate == GeneratingCommand.generate:
raise AttributeError('No GeneratingCommand.generate override')
+ # TODO: Stop looking like a dictionary because we don't obey the semantics
+ # N.B.: Does not use Python 2 dict copy semantics
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
version = self.command.protocol_version
if version == 2:
- iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
- if self.distributed and self.type == 'streaming':
+ iteritems = ifilter(lambda name_value1: name_value1[0] != 'distributed', iteritems)
+ if not self.distributed and self.type == 'streaming':
iteritems = imap(
- lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
+ lambda name_value: (name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]), iteritems)
return iteritems
+ # N.B.: Does not use Python 3 dict view semantics
+ if not six.PY2:
+ items = iteritems
+
pass
# endregion
diff --git a/dist/escu/bin/splunklib/searchcommands/internals.py b/dist/escu/bin/splunklib/searchcommands/internals.py
old mode 100755
new mode 100644
index be57703eb6..85f9e0fe1e
--- a/dist/escu/bin/splunklib/searchcommands/internals.py
+++ b/dist/escu/bin/splunklib/searchcommands/internals.py
@@ -14,45 +14,68 @@
# License for the specific language governing permissions and limitations
# under the License.
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
+from io import TextIOWrapper
from collections import deque, namedtuple
+from splunklib import six
try:
from collections import OrderedDict # must be python 2.7
except ImportError:
from ..ordereddict import OrderedDict
-from cStringIO import StringIO
-from itertools import chain, imap
+from splunklib.six.moves import StringIO
+from itertools import chain
+from splunklib.six.moves import map as imap
from json import JSONDecoder, JSONEncoder
from json.encoder import encode_basestring_ascii as json_encode_string
-from urllib import unquote
+from splunklib.six.moves import urllib
import csv
import gzip
import os
import re
import sys
+import warnings
from . import environment
csv.field_size_limit(10485760) # The default value is 128KB; upping to 10MB. See SPL-12117 for background on this issue
-if sys.platform == 'win32':
- # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in
- # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that
- # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption.
- from platform import python_implementation
- implementation = python_implementation()
- fileno = sys.stdout.fileno()
- if implementation == 'PyPy':
- sys.stdout = os.fdopen(fileno, 'wb', 0)
- else:
- from msvcrt import setmode
- setmode(fileno, os.O_BINARY)
+
+def set_binary_mode(fh):
+ """ Helper method to set up binary mode for file handles.
+ Emphasis being sys.stdin, sys.stdout, sys.stderr.
+ For python3, we want to return .buffer
+ For python2+windows we want to set os.O_BINARY
+ """
+ typefile = TextIOWrapper if sys.version_info >= (3, 0) else file
+ # check for file handle
+ if not isinstance(fh, typefile):
+ return fh
+
+ # check for python3 and buffer
+ if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'):
+ return fh.buffer
+ # check for python3
+ elif sys.version_info >= (3, 0):
+ pass
+ # check for windows python2. SPL-175233 -- python3 stdout is already binary
+ elif sys.platform == 'win32':
+ # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in
+ # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that
+ # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption.
+ from platform import python_implementation
+ implementation = python_implementation()
+ if implementation == 'PyPy':
+ return os.fdopen(fh.fileno(), 'wb', 0)
+ else:
+ import msvcrt
+ msvcrt.setmode(fh.fileno(), os.O_BINARY)
+ return fh
class CommandLineParser(object):
- """ Parses the arguments to a search command.
+ r""" Parses the arguments to a search command.
A search command line is described by the following syntax.
@@ -210,7 +233,7 @@ class CommandLineParser(object):
_escaped_character_re = re.compile(r'(\\.|""|[\\"])')
- _fieldnames_re = re.compile(r"""("(?:\\.|""|[^"])+"|(?:\\.|[^\s"])+)""")
+ _fieldnames_re = re.compile(r"""("(?:\\.|""|[^"\\])+"|(?:\\.|[^\s"])+)""")
_options_re = re.compile(r"""
# Captures a set of name/value pairs when used with re.finditer
@@ -243,7 +266,7 @@ class ConfigurationSettingsType(type):
"""
def __new__(mcs, module, name, bases):
- mcs = super(ConfigurationSettingsType, mcs).__new__(mcs, name, bases, {})
+ mcs = super(ConfigurationSettingsType, mcs).__new__(mcs, str(name), bases, {})
return mcs
def __init__(cls, module, name, bases):
@@ -264,10 +287,10 @@ class ConfigurationSettingsType(type):
return value
specification = namedtuple(
- b'ConfigurationSettingSpecification', (
- b'type',
- b'constraint',
- b'supporting_protocols'))
+ 'ConfigurationSettingSpecification', (
+ 'type',
+ 'constraint',
+ 'supporting_protocols'))
# P1 [ ] TODO: Review ConfigurationSettingsType.specification_matrix for completeness and correctness
@@ -294,7 +317,7 @@ class ConfigurationSettingsType(type):
supporting_protocols=[1]),
'maxinputs': specification(
type=int,
- constraint=lambda value: 0 <= value <= sys.maxint,
+ constraint=lambda value: 0 <= value <= six.MAXSIZE,
supporting_protocols=[2]),
'overrides_timeorder': specification(
type=bool,
@@ -321,22 +344,24 @@ class ConfigurationSettingsType(type):
constraint=None,
supporting_protocols=[1]),
'streaming_preop': specification(
- type=(bytes, unicode),
+ type=(bytes, six.text_type),
constraint=None,
supporting_protocols=[1, 2]),
'type': specification(
- type=(bytes, unicode),
- constraint=lambda value: value in ('eventing', 'reporting', 'streaming'),
+ type=(bytes, six.text_type),
+ constraint=lambda value: value in ('events', 'reporting', 'streaming'),
supporting_protocols=[2])}
class CsvDialect(csv.Dialect):
""" Describes the properties of Splunk CSV streams """
- delimiter = b','
- quotechar = b'"'
+ delimiter = ','
+ quotechar = '"'
doublequote = True
skipinitialspace = False
- lineterminator = b'\r\n'
+ lineterminator = '\r\n'
+ if sys.version_info >= (3, 0) and sys.platform == 'win32':
+ lineterminator = '\n'
quoting = csv.QUOTE_MINIMAL
@@ -344,8 +369,9 @@ class InputHeader(dict):
""" Represents a Splunk input header as a collection of name/value pairs.
"""
+
def __str__(self):
- return '\n'.join([name + ':' + value for name, value in self.iteritems()])
+ return '\n'.join([name + ':' + value for name, value in six.iteritems(self)])
def read(self, ifile):
""" Reads an input header from an input file.
@@ -366,15 +392,16 @@ class InputHeader(dict):
# start of a new item
if name is not None:
self[name] = value[:-1] # value sans trailing newline
- name, value = item[0], unquote(item[1])
+ name, value = item[0], urllib.parse.unquote(item[1])
elif name is not None:
# continuation of the current item
- value += unquote(line)
+ value += urllib.parse.unquote(line)
- if name is not None: self[name] = value[:-1] if value[-1] == '\n' else value
+ if name is not None:
+ self[name] = value[:-1] if value[-1] == '\n' else value
-Message = namedtuple(b'Message', (b'type', b'text'))
+Message = namedtuple('Message', ('type', 'text'))
class MetadataDecoder(JSONDecoder):
@@ -392,7 +419,7 @@ class MetadataDecoder(JSONDecoder):
while len(stack):
instance, member_name, dictionary = stack.popleft()
- for name, value in dictionary.iteritems():
+ for name, value in six.iteritems(dictionary):
if isinstance(value, dict):
stack.append((dictionary, name, value))
@@ -468,7 +495,7 @@ class RecordWriter(object):
def __init__(self, ofile, maxresultrows=None):
self._maxresultrows = 50000 if maxresultrows is None else maxresultrows
- self._ofile = ofile
+ self._ofile = set_binary_mode(ofile)
self._fieldnames = None
self._buffer = StringIO()
@@ -479,8 +506,8 @@ class RecordWriter(object):
self._inspector = OrderedDict()
self._chunk_count = 0
- self._record_count = 0
- self._total_record_count = 0L
+ self._pending_record_count = 0
+ self._committed_record_count = 0
@property
def is_flushed(self):
@@ -496,7 +523,37 @@ class RecordWriter(object):
@ofile.setter
def ofile(self, value):
- self._ofile = value
+ self._ofile = set_binary_mode(value)
+
+ @property
+ def pending_record_count(self):
+ return self._pending_record_count
+
+ @property
+ def _record_count(self):
+ warnings.warn(
+ "_record_count will be deprecated soon. Use pending_record_count instead.",
+ PendingDeprecationWarning
+ )
+ return self.pending_record_count
+
+ @property
+ def committed_record_count(self):
+ return self._committed_record_count
+
+ @property
+ def _total_record_count(self):
+ warnings.warn(
+ "_total_record_count will be deprecated soon. Use committed_record_count instead.",
+ PendingDeprecationWarning
+ )
+ return self.committed_record_count
+
+ def write(self, data):
+ bytes_type = bytes if sys.version_info >= (3, 0) else str
+ if not isinstance(data, bytes_type):
+ data = data.encode('utf-8')
+ self.ofile.write(data)
def flush(self, finished=None, partial=None):
assert finished is None or isinstance(finished, bool)
@@ -520,11 +577,10 @@ class RecordWriter(object):
write_record(record)
def _clear(self):
- self._buffer.reset()
+ self._buffer.seek(0)
self._buffer.truncate()
self._inspector.clear()
- self._record_count = 0
- self._flushed = False
+ self._pending_record_count = 0
def _ensure_validity(self):
if self._finished is True:
@@ -536,9 +592,8 @@ class RecordWriter(object):
fieldnames = self._fieldnames
if fieldnames is None:
- self._fieldnames = fieldnames = record.keys()
- value_list = imap(lambda fn: unicode(fn).encode('utf-8'), fieldnames)
- value_list = imap(lambda fn: (fn, b'__mv_' + fn), value_list)
+ self._fieldnames = fieldnames = list(record.keys())
+ value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames)
self._writerow(list(chain.from_iterable(value_list)))
get_value = record.get
@@ -561,14 +616,14 @@ class RecordWriter(object):
if len(value) > 1:
value_list = value
- sv = b''
- mv = b'$'
+ sv = ''
+ mv = '$'
for value in value_list:
if value is None:
- sv += b'\n'
- mv += b'$;$'
+ sv += '\n'
+ mv += '$;$'
continue
value_t = type(value)
@@ -577,17 +632,17 @@ class RecordWriter(object):
if value_t is bool:
value = str(value.real)
- elif value_t is unicode:
- value = value.encode('utf-8', errors='backslashreplace')
- elif value_t is int or value_t is long or value_t is float or value_t is complex:
+ elif value_t is six.text_type:
+ value = value
+ elif isinstance(value, six.integer_types) or value_t is float or value_t is complex:
value = str(value)
elif issubclass(value_t, (dict, list, tuple)):
value = str(''.join(RecordWriter._iterencode_json(value, 0)))
else:
value = repr(value).encode('utf-8', errors='backslashreplace')
- sv += value + b'\n'
- mv += value.replace(b'$', b'$$') + b'$;$'
+ sv += value + '\n'
+ mv += value.replace('$', '$$') + '$;$'
values += (sv[:-1], mv[:-2])
continue
@@ -603,11 +658,13 @@ class RecordWriter(object):
values += (value, None)
continue
- if value_t is unicode:
- values += (value.encode('utf-8', errors='backslashreplace'), None)
+ if value_t is six.text_type:
+ if six.PY2:
+ value = value.encode('utf-8')
+ values += (value, None)
continue
- if value_t is int or value_t is long or value_t is float or value_t is complex:
+ if isinstance(value, six.integer_types) or value_t is float or value_t is complex:
values += (str(value), None)
continue
@@ -615,12 +672,12 @@ class RecordWriter(object):
values += (str(''.join(RecordWriter._iterencode_json(value, 0))), None)
continue
- values += (repr(value).encode('utf-8', errors='backslashreplace'), None)
+ values += (repr(value), None)
self._writerow(values)
- self._record_count += 1
+ self._pending_record_count += 1
- if self._record_count >= self._maxresultrows:
+ if self.pending_record_count >= self._maxresultrows:
self.flush(partial=True)
try:
@@ -657,10 +714,9 @@ class RecordWriterV1(RecordWriter):
RecordWriter.flush(self, finished, partial) # validates arguments and the state of this instance
- if self._record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector):
+ if self.pending_record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector):
messages = self._inspector.get('messages')
- write = self._ofile.write
if self._chunk_count == 0:
@@ -672,12 +728,12 @@ class RecordWriterV1(RecordWriter):
message_level = RecordWriterV1._message_level.get
for level, text in messages:
- write(message_level(level, level))
- write('=')
- write(text)
- write('\r\n')
+ self.write(message_level(level, level))
+ self.write('=')
+ self.write(text)
+ self.write('\r\n')
- write('\r\n')
+ self.write('\r\n')
elif messages is not None:
@@ -695,10 +751,10 @@ class RecordWriterV1(RecordWriter):
for level, text in messages:
print(level, text, file=stderr)
- write(self._buffer.getvalue())
- self._clear()
+ self.write(self._buffer.getvalue())
self._chunk_count += 1
- self._total_record_count += self._record_count
+ self._committed_record_count += self.pending_record_count
+ self._clear()
self._finished = finished is True
@@ -716,44 +772,43 @@ class RecordWriterV2(RecordWriter):
def flush(self, finished=None, partial=None):
RecordWriter.flush(self, finished, partial) # validates arguments and the state of this instance
+
+ if partial or not finished:
+ # Don't flush partial chunks, since the SCP v2 protocol does not
+ # provide a way to send partial chunks yet.
+ return
+
+ if not self.is_flushed:
+ self.write_chunk(finished=True)
+
+ def write_chunk(self, finished=None):
inspector = self._inspector
+ self._committed_record_count += self.pending_record_count
+ self._chunk_count += 1
- if self._flushed is False:
+ # TODO: DVPL-6448: splunklib.searchcommands | Add support for partial: true when it is implemented in
+ # ChunkedExternProcessor (See SPL-103525)
+ #
+ # We will need to replace the following block of code with this block:
+ #
+ # metadata = [item for item in (('inspector', inspector), ('finished', finished), ('partial', partial))]
+ #
+ # if partial is True:
+ # finished = False
- self._total_record_count += self._record_count
- self._chunk_count += 1
+ if len(inspector) == 0:
+ inspector = None
- # TODO: DVPL-6448: splunklib.searchcommands | Add support for partial: true when it is implemented in
- # ChunkedExternProcessor (See SPL-103525)
- #
- # We will need to replace the following block of code with this block:
- #
- # metadata = [
- # ('inspector', self._inspector if len(self._inspector) else None),
- # ('finished', finished),
- # ('partial', partial)]
-
- if len(inspector) == 0:
- inspector = None
-
- if partial is True:
- finished = False
-
- metadata = [item for item in ('inspector', inspector), ('finished', finished)]
- self._write_chunk(metadata, self._buffer.getvalue())
- self._clear()
-
- elif finished is True:
- self._write_chunk((('finished', True),), '')
-
- self._finished = finished is True
+ metadata = [item for item in (('inspector', inspector), ('finished', finished))]
+ self._write_chunk(metadata, self._buffer.getvalue())
+ self._clear()
def write_metadata(self, configuration):
self._ensure_validity()
- metadata = chain(configuration.iteritems(), (('inspector', self._inspector if self._inspector else None),))
+ metadata = chain(six.iteritems(configuration), (('inspector', self._inspector if self._inspector else None),))
self._write_chunk(metadata, '')
- self._ofile.write('\n')
+ self.write('\n')
self._clear()
def write_metric(self, name, value):
@@ -761,26 +816,29 @@ class RecordWriterV2(RecordWriter):
self._inspector['metric.' + name] = value
def _clear(self):
- RecordWriter._clear(self)
+ super(RecordWriterV2, self)._clear()
self._fieldnames = None
def _write_chunk(self, metadata, body):
if metadata:
metadata = str(''.join(self._iterencode_json(dict([(n, v) for n, v in metadata if v is not None]), 0)))
+ if sys.version_info >= (3, 0):
+ metadata = metadata.encode('utf-8')
metadata_length = len(metadata)
else:
metadata_length = 0
+ if sys.version_info >= (3, 0):
+ body = body.encode('utf-8')
body_length = len(body)
if not (metadata_length > 0 or body_length > 0):
return
- start_line = b'chunked 1.0,' + bytes(metadata_length) + b',' + bytes(body_length) + b'\n'
- write = self._ofile.write
- write(start_line)
- write(metadata)
- write(body)
+ start_line = 'chunked 1.0,%s,%s\n' % (metadata_length, body_length)
+ self.write(start_line)
+ self.write(metadata)
+ self.write(body)
self._ofile.flush()
- self._flushed = False
+ self._flushed = True
diff --git a/dist/escu/bin/splunklib/searchcommands/reporting_command.py b/dist/escu/bin/splunklib/searchcommands/reporting_command.py
old mode 100755
new mode 100644
index c856ee13e1..947086197a
--- a/dist/escu/bin/splunklib/searchcommands/reporting_command.py
+++ b/dist/escu/bin/splunklib/searchcommands/reporting_command.py
@@ -23,6 +23,7 @@ from .decorators import ConfigurationSetting, Option
from .streaming_command import StreamingCommand
from .search_command import SearchCommand
from .validators import Set
+from splunklib import six
class ReportingCommand(SearchCommand):
@@ -93,7 +94,7 @@ class ReportingCommand(SearchCommand):
self._configuration.streaming_preop = ' '.join(streaming_preop)
return
- raise RuntimeError('Unrecognized reporting command phase: {}'.format(json_encode_string(unicode(phase))))
+ raise RuntimeError('Unrecognized reporting command phase: {}'.format(json_encode_string(six.text_type(phase))))
def reduce(self, records):
""" Override this method to produce a reporting data structure.
@@ -252,7 +253,7 @@ class ReportingCommand(SearchCommand):
cls._requires_preop = False
return
- f = vars(command)[b'map'] # Function backing the map method
+ f = vars(command)['map'] # Function backing the map method
# EXPLANATION OF PREVIOUS STATEMENT: There is no way to add custom attributes to methods. See [Why does
# setattr fail on a method](http://stackoverflow.com/questions/7891277/why-does-setattr-fail-on-a-bound-method) for a discussion of this issue.
@@ -265,7 +266,7 @@ class ReportingCommand(SearchCommand):
# Create new StreamingCommand.ConfigurationSettings class
- module = command.__module__ + b'.' + command.__name__ + b'.map'
+ module = command.__module__ + '.' + command.__name__ + '.map'
name = b'ConfigurationSettings'
bases = (StreamingCommand.ConfigurationSettings,)
diff --git a/dist/escu/bin/splunklib/searchcommands/search_command.py b/dist/escu/bin/splunklib/searchcommands/search_command.py
old mode 100755
new mode 100644
index 2f6cb086be..7383a5efa4
--- a/dist/escu/bin/splunklib/searchcommands/search_command.py
+++ b/dist/escu/bin/splunklib/searchcommands/search_command.py
@@ -19,22 +19,30 @@ from __future__ import absolute_import, division, print_function, unicode_litera
# Absolute imports
from collections import namedtuple
+
+import io
+
try:
from collections import OrderedDict # must be python 2.7
except ImportError:
from ..ordereddict import OrderedDict
from copy import deepcopy
-from cStringIO import StringIO
-from itertools import chain, ifilter, imap, islice, izip
-from logging import _levelNames, getLevelName, getLogger
+from splunklib.six.moves import StringIO
+from itertools import chain, islice
+from splunklib.six.moves import filter as ifilter, map as imap, zip as izip
+from splunklib import six
+if six.PY2:
+ from logging import _levelNames, getLevelName, getLogger
+else:
+ from logging import _nameToLevel as _levelNames, getLevelName, getLogger
try:
from shutil import make_archive
except ImportError:
# Used for recording, skip on python 2.6
pass
from time import time
-from urllib import unquote
-from urlparse import urlsplit
+from splunklib.six.moves.urllib.parse import unquote
+from splunklib.six.moves.urllib.parse import urlsplit
from warnings import warn
from xml.etree import ElementTree
@@ -47,7 +55,7 @@ import traceback
# Relative imports
-from . internals import (
+from .internals import (
CommandLineParser,
CsvDialect,
InputHeader,
@@ -63,6 +71,7 @@ from . internals import (
from . import Boolean, Option, environment
from ..client import Service
+
# ----------------------------------------------------------------------------------------------------------------------
# P1 [ ] TODO: Log these issues against ChunkedExternProcessor
@@ -89,6 +98,7 @@ class SearchCommand(object):
""" Represents a custom search command.
"""
+
def __init__(self):
# Variables that may be used, but not altered by derived classes
@@ -150,7 +160,7 @@ class SearchCommand(object):
def logging_level(self, value):
if value is None:
value = self._default_logging_level
- if isinstance(value, (bytes, unicode)):
+ if isinstance(value, (bytes, six.text_type)):
try:
level = _levelNames[value.upper()]
except KeyError:
@@ -246,7 +256,7 @@ class SearchCommand(object):
invocation.
:return: Search results info:const:`None`, if the search results info file associated with the command
- invocation is inaccessible.
+ invocation is inaccessible.
:rtype: SearchResultsInfo or NoneType
"""
@@ -269,10 +279,10 @@ class SearchCommand(object):
path = os.path.join(dispatch_dir, 'info.csv')
try:
- with open(path, 'rb') as f:
+ with io.open(path, 'r') as f:
reader = csv.reader(f, dialect=CsvDialect)
- fields = reader.next()
- values = reader.next()
+ fields = next(reader)
+ values = next(reader)
except IOError as error:
if error.errno == 2:
self.logger.error('Search results info file {} does not exist.'.format(json_encode_string(path)))
@@ -290,7 +300,7 @@ class SearchCommand(object):
except ValueError:
return value
- info = ObjectView(dict(imap(lambda (f, v): (convert_field(f), convert_value(v)), izip(fields, values))))
+ info = ObjectView(dict(imap(lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values))))
try:
count_map = info.countMap
@@ -307,7 +317,7 @@ class SearchCommand(object):
except AttributeError:
pass
else:
- messages = ifilter(lambda (t, m): t or m, izip(msg_type.split('\n'), msg_text.split('\n')))
+ messages = ifilter(lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n')))
info.msg = [Message(message) for message in messages]
del info.msgType
@@ -328,6 +338,7 @@ class SearchCommand(object):
specifying this pair of configuration settings in commands.conf:
.. code-block:: python
+
enableheader = true
requires_srinfo = true
@@ -335,8 +346,8 @@ class SearchCommand(object):
:code:`requires_srinfo` setting is false by default. Hence, you must set it.
:return: :class:`splunklib.client.Service`, if :code:`enableheader` and :code:`requires_srinfo` are both
- :code:`true`. Otherwise, if either :code:`enableheader` or :code:`requires_srinfo` are :code:`false`, a value
- of :code:`None` is returned.
+ :code:`true`. Otherwise, if either :code:`enableheader` or :code:`requires_srinfo` are :code:`false`, a value
+ of :code:`None` is returned.
"""
if self._service is not None:
@@ -387,7 +398,7 @@ class SearchCommand(object):
:return: :const:`None`
"""
- self._record_writer.flush(partial=True)
+ self._record_writer.flush(finished=False)
def prepare(self):
""" Prepare for execution.
@@ -443,7 +454,7 @@ class SearchCommand(object):
def _map(metadata_map):
metadata = {}
- for name, value in metadata_map.iteritems():
+ for name, value in six.iteritems(metadata_map):
if isinstance(value, dict):
value = _map(value)
else:
@@ -493,7 +504,7 @@ class SearchCommand(object):
'username':
(lambda v: v.ppc_user, lambda s: s.search_results_info)}}
- _MetadataSource = namedtuple(b'Source', (b'argv', b'input_header', b'search_results_info'))
+ _MetadataSource = namedtuple('Source', ('argv', 'input_header', 'search_results_info'))
def _prepare_protocol_v1(self, argv, ifile, ofile):
@@ -580,7 +591,7 @@ class SearchCommand(object):
ifile = self._prepare_protocol_v1(argv, ifile, ofile)
self._record_writer.write_record(dict(
- (n, ','.join(v) if isinstance(v, (list, tuple)) else v) for n, v in self._configuration.iteritems()))
+ (n, ','.join(v) if isinstance(v, (list, tuple)) else v) for n, v in six.iteritems(self._configuration)))
self.finish()
elif argv[1] == '__EXECUTE__':
@@ -608,7 +619,7 @@ class SearchCommand(object):
raise RuntimeError(message)
except (SyntaxError, ValueError) as error:
- self.write_error(unicode(error))
+ self.write_error(six.text_type(error))
self.flush()
exit(0)
@@ -645,7 +656,7 @@ class SearchCommand(object):
# noinspection PyBroadException
try:
debug('Reading metadata')
- metadata, body = self._read_chunk(ifile)
+ metadata, body = self._read_chunk(self._as_binary_stream(ifile))
action = getattr(metadata, 'action', None)
@@ -695,9 +706,10 @@ class SearchCommand(object):
for arg in args:
result = arg.split('=', 1)
if len(result) == 1:
- self.fieldnames.append(result[0])
+ self.fieldnames.append(str(result[0]))
else:
name, value = result
+ name = str(name)
try:
option = self.options[name]
except KeyError:
@@ -723,7 +735,7 @@ class SearchCommand(object):
if error_count > 0:
exit(1)
- debug(' command: %s', unicode(self))
+ debug(' command: %s', six.text_type(self))
debug('Preparing for execution')
self.prepare()
@@ -741,7 +753,7 @@ class SearchCommand(object):
setattr(info, attr, [arg for arg in getattr(info, attr) if not arg.startswith('record=')])
metadata = MetadataEncoder().encode(self._metadata)
- ifile.record('chunked 1.0,', unicode(len(metadata)), ',0\n', metadata)
+ ifile.record('chunked 1.0,', six.text_type(len(metadata)), ',0\n', metadata)
if self.show_configuration:
self.write_info(self.name + ' command configuration: ' + str(self._configuration))
@@ -764,7 +776,6 @@ class SearchCommand(object):
# noinspection PyBroadException
try:
debug('Executing under protocol_version=2')
- self._records = self._records_protocol_v2
self._metadata.action = 'execute'
self._execute(ifile, None)
except SystemExit:
@@ -798,15 +809,15 @@ class SearchCommand(object):
:param name: Name of the metric.
:type name: basestring
- :param value: A 4-tuple containing the value of metric :param:`name` where
+ :param value: A 4-tuple containing the value of metric ``name`` where
value[0] = Elapsed seconds or :const:`None`.
value[1] = Number of invocations or :const:`None`.
value[2] = Input count or :const:`None`.
value[3] = Output count or :const:`None`.
- The :data:`SearchMetric` type provides a convenient encapsulation of :param:`value`.
- The :data:`SearchMetric` type provides a convenient encapsulation of :param:`value`.
+ The :data:`SearchMetric` type provides a convenient encapsulation of ``value``.
+ The :data:`SearchMetric` type provides a convenient encapsulation of ``value``.
:return: :const:`None`.
@@ -821,6 +832,8 @@ class SearchCommand(object):
_encoded_value = re.compile(r'\$(?P- (?:\$\$|[^$])*)\$(?:;|$)') # matches a single value in an encoded list
+ # Note: Subclasses must override this method so that it can be called
+ # called as self._execute(ifile, None)
def _execute(self, ifile, process):
""" Default processing loop
@@ -834,22 +847,38 @@ class SearchCommand(object):
:rtype: NoneType
"""
- self._record_writer.write_records(process(self._records(ifile)))
- self.finish()
+ if self.protocol_version == 1:
+ self._record_writer.write_records(process(self._records(ifile)))
+ self.finish()
+ else:
+ assert self._protocol_version == 2
+ self._execute_v2(ifile, process)
@staticmethod
- def _read_chunk(ifile):
+ def _as_binary_stream(ifile):
+ naught = ifile.read(0)
+ if isinstance(naught, bytes):
+ return ifile
- # noinspection PyBroadException
try:
- header = ifile.readline()
+ return ifile.buffer
+ except AttributeError as error:
+ raise RuntimeError('Failed to get underlying buffer: {}'.format(error))
+
+ @staticmethod
+ def _read_chunk(istream):
+ # noinspection PyBroadException
+ assert isinstance(istream.read(0), six.binary_type), 'Stream must be binary'
+
+ try:
+ header = istream.readline()
except Exception as error:
raise RuntimeError('Failed to read transport header: {}'.format(error))
if not header:
return None
- match = SearchCommand._header.match(header)
+ match = SearchCommand._header.match(six.ensure_str(header))
if match is None:
raise RuntimeError('Failed to parse transport header: {}'.format(header))
@@ -859,35 +888,39 @@ class SearchCommand(object):
body_length = int(body_length)
try:
- metadata = ifile.read(metadata_length)
+ metadata = istream.read(metadata_length)
except Exception as error:
raise RuntimeError('Failed to read metadata of length {}: {}'.format(metadata_length, error))
decoder = MetadataDecoder()
try:
- metadata = decoder.decode(metadata)
+ metadata = decoder.decode(six.ensure_str(metadata))
except Exception as error:
raise RuntimeError('Failed to parse metadata of length {}: {}'.format(metadata_length, error))
# if body_length <= 0:
# return metadata, ''
+ body = ""
try:
- body = ifile.read(body_length)
+ if body_length > 0:
+ body = istream.read(body_length)
except Exception as error:
raise RuntimeError('Failed to read body of length {}: {}'.format(body_length, error))
- return metadata, body
+ return metadata, six.ensure_str(body)
_header = re.compile(r'chunked\s+1.0\s*,\s*(\d+)\s*,\s*(\d+)\s*\n')
def _records_protocol_v1(self, ifile):
+ return self._read_csv_records(ifile)
+ def _read_csv_records(self, ifile):
reader = csv.reader(ifile, dialect=CsvDialect)
try:
- fieldnames = reader.next()
+ fieldnames = next(reader)
except StopIteration:
return
@@ -908,51 +941,36 @@ class SearchCommand(object):
record[fieldname] = value
yield record
- def _records_protocol_v2(self, ifile):
+ def _execute_v2(self, ifile, process):
+ istream = self._as_binary_stream(ifile)
while True:
- result = self._read_chunk(ifile)
+ result = self._read_chunk(istream)
if not result:
return
metadata, body = result
action = getattr(metadata, 'action', None)
-
if action != 'execute':
raise RuntimeError('Expected execute action, not {}'.format(action))
- finished = getattr(metadata, 'finished', False)
+ self._finished = getattr(metadata, 'finished', False)
self._record_writer.is_flushed = False
- if len(body) > 0:
- reader = csv.reader(StringIO(body), dialect=CsvDialect)
+ self._execute_chunk_v2(process, result)
- try:
- fieldnames = reader.next()
- except StopIteration:
- return
+ self._record_writer.write_chunk(finished=self._finished)
- mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')])
+ def _execute_chunk_v2(self, process, chunk):
+ metadata, body = chunk
- if len(mv_fieldnames) == 0:
- for values in reader:
- yield OrderedDict(izip(fieldnames, values))
- else:
- for values in reader:
- record = OrderedDict()
- for fieldname, value in izip(fieldnames, values):
- if fieldname.startswith('__mv_'):
- if len(value) > 0:
- record[mv_fieldnames[fieldname]] = self._decode_list(value)
- elif fieldname not in record:
- record[fieldname] = value
- yield record
-
- if finished:
+ if len(body) <= 0:
return
- self.flush()
+ records = self._read_csv_records(StringIO(body))
+ self._record_writer.write_records(process(records))
+
def _report_unexpected_error(self):
@@ -1003,7 +1021,8 @@ class SearchCommand(object):
:return: String representation of this instance
"""
- text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems()))
+ #text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems()))
+ text = ', '.join(['{}={}'.format(name, json_encode_string(six.text_type(value))) for (name, value) in six.iteritems(self)])
return text
# region Methods
@@ -1022,20 +1041,27 @@ class SearchCommand(object):
"""
return
+ # TODO: Stop looking like a dictionary because we don't obey the semantics
+ # N.B.: Does not use Python 2 dict copy semantics
def iteritems(self):
definitions = type(self).configuration_setting_definitions
version = self.command.protocol_version
return ifilter(
- lambda (name, value): value is not None, imap(
+ lambda name_value1: name_value1[1] is not None, imap(
lambda setting: (setting.name, setting.__get__(self)), ifilter(
lambda setting: setting.is_supported_by_protocol(version), definitions)))
+ # N.B.: Does not use Python 3 dict view semantics
+ if not six.PY2:
+ items = iteritems
+
pass # endregion
pass # endregion
-SearchMetric = namedtuple(b'SearchMetric', (b'elapsed_seconds', b'invocation_count', b'input_count', b'output_count'))
+SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count'))
+
def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None):
@@ -1065,7 +1091,7 @@ def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys
**Example**
- .. code-block:: python
+ .. code-block:: python
:linenos:
#!/usr/bin/env python
@@ -1081,7 +1107,7 @@ def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys
**Example**
- .. code-block:: python
+ .. code-block:: python
:linenos:
from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option, validators
diff --git a/dist/escu/bin/splunklib/searchcommands/streaming_command.py b/dist/escu/bin/splunklib/searchcommands/streaming_command.py
old mode 100755
new mode 100644
index 12e9f035d8..fa075edb19
--- a/dist/escu/bin/splunklib/searchcommands/streaming_command.py
+++ b/dist/escu/bin/splunklib/searchcommands/streaming_command.py
@@ -16,7 +16,8 @@
from __future__ import absolute_import, division, print_function, unicode_literals
-from itertools import ifilter, imap
+from splunklib import six
+from splunklib.six.moves import map as imap, filter as ifilter
from .decorators import ConfigurationSetting
from .search_command import SearchCommand
@@ -172,17 +173,23 @@ class StreamingCommand(SearchCommand):
raise AttributeError('No StreamingCommand.stream override')
return
+ # TODO: Stop looking like a dictionary because we don't obey the semantics
+ # N.B.: Does not use Python 2 dict copy semantics
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
version = self.command.protocol_version
if version == 1:
if self.required_fields is None:
- iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
+ iteritems = ifilter(lambda name_value: name_value[0] != 'clear_required_fields', iteritems)
else:
- iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
- if self.distributed:
+ iteritems = ifilter(lambda name_value2: name_value2[0] != 'distributed', iteritems)
+ if not self.distributed:
iteritems = imap(
- lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
+ lambda name_value1: (name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]), iteritems)
return iteritems
+ # N.B.: Does not use Python 3 dict view semantics
+ if not six.PY2:
+ items = iteritems
+
# endregion
diff --git a/dist/escu/bin/splunklib/searchcommands/validators.py b/dist/escu/bin/splunklib/searchcommands/validators.py
old mode 100755
new mode 100644
index 9b9fee30b5..0278fbd599
--- a/dist/escu/bin/splunklib/searchcommands/validators.py
+++ b/dist/escu/bin/splunklib/searchcommands/validators.py
@@ -18,11 +18,13 @@ from __future__ import absolute_import, division, print_function, unicode_litera
from json.encoder import encode_basestring_ascii as json_encode_string
from collections import namedtuple
-from cStringIO import StringIO
+from splunklib.six.moves import StringIO
from io import open
import csv
import os
import re
+from splunklib import six
+from splunklib.six.moves import getcwd
class Validator(object):
@@ -58,7 +60,7 @@ class Boolean(Validator):
def __call__(self, value):
if not (value is None or isinstance(value, bool)):
- value = unicode(value).lower()
+ value = six.text_type(value).lower()
if value not in Boolean.truth_values:
raise ValueError('Unrecognized truth value: {0}'.format(value))
value = Boolean.truth_values[value]
@@ -79,9 +81,9 @@ class Code(Validator):
def __init__(self, mode='eval'):
"""
:param mode: Specifies what kind of code must be compiled; it can be :const:`'exec'`, if source consists of a
- sequence of statements, :const:`'eval'`, if it consists of a single expression, or :const:`'single'` if it
- consists of a single interactive statement. In the latter case, expression statements that evaluate to
- something other than :const:`None` will be printed.
+ sequence of statements, :const:`'eval'`, if it consists of a single expression, or :const:`'single'` if it
+ consists of a single interactive statement. In the latter case, expression statements that evaluate to
+ something other than :const:`None` will be printed.
:type mode: unicode or bytes
"""
@@ -91,14 +93,19 @@ class Code(Validator):
if value is None:
return None
try:
- return Code.object(compile(value, 'string', self._mode), unicode(value))
+ return Code.object(compile(value, 'string', self._mode), six.text_type(value))
except (SyntaxError, TypeError) as error:
- raise ValueError(error.message)
+ if six.PY2:
+ message = error.message
+ else:
+ message = str(error)
+
+ six.raise_from(ValueError(message), error)
def format(self, value):
return None if value is None else value.source
- object = namedtuple(b'Code', (b'object', 'source'))
+ object = namedtuple('Code', ('object', 'source'))
class Fieldname(Validator):
@@ -109,7 +116,7 @@ class Fieldname(Validator):
def __call__(self, value):
if value is not None:
- value = unicode(value)
+ value = six.text_type(value)
if Fieldname.pattern.match(value) is None:
raise ValueError('Illegal characters in fieldname: {}'.format(value))
return value
@@ -132,7 +139,7 @@ class File(Validator):
if value is None:
return value
- path = unicode(value)
+ path = six.text_type(value)
if not os.path.isabs(path):
path = os.path.join(self.directory, path)
@@ -149,7 +156,7 @@ class File(Validator):
return None if value is None else value.name
_var_run_splunk = os.path.join(
- os.environ['SPLUNK_HOME'] if 'SPLUNK_HOME' in os.environ else os.getcwdu(), 'var', 'run', 'splunk')
+ os.environ['SPLUNK_HOME'] if 'SPLUNK_HOME' in os.environ else getcwd(), 'var', 'run', 'splunk')
class Integer(Validator):
@@ -183,7 +190,10 @@ class Integer(Validator):
if value is None:
return None
try:
- value = long(value)
+ if six.PY2:
+ value = long(value)
+ else:
+ value = int(value)
except ValueError:
raise ValueError('Expected integer value, not {}'.format(json_encode_string(value)))
@@ -191,7 +201,7 @@ class Integer(Validator):
return value
def format(self, value):
- return None if value is None else unicode(long(value))
+ return None if value is None else six.text_type(int(value))
class Duration(Validator):
@@ -244,10 +254,10 @@ class List(Validator):
class Dialect(csv.Dialect):
""" Describes the properties of list option values. """
strict = True
- delimiter = b','
- quotechar = b'"'
+ delimiter = str(',')
+ quotechar = str('"')
doublequote = True
- lineterminator = b'\n'
+ lineterminator = str('\n')
skipinitialspace = True
quoting = csv.QUOTE_MINIMAL
@@ -262,7 +272,7 @@ class List(Validator):
return value
try:
- value = csv.reader([value], self.Dialect).next()
+ value = next(csv.reader([value], self.Dialect))
except csv.Error as error:
raise ValueError(error)
@@ -297,7 +307,7 @@ class Map(Validator):
if value is None:
return None
- value = unicode(value)
+ value = six.text_type(value)
if value not in self.membership:
raise ValueError('Unrecognized value: {0}'.format(value))
@@ -305,7 +315,7 @@ class Map(Validator):
return self.membership[value]
def format(self, value):
- return None if value is None else self.membership.keys()[self.membership.values().index(value)]
+ return None if value is None else list(self.membership.keys())[list(self.membership.values()).index(value)]
class Match(Validator):
@@ -313,19 +323,19 @@ class Match(Validator):
"""
def __init__(self, name, pattern, flags=0):
- self.name = unicode(name)
+ self.name = six.text_type(name)
self.pattern = re.compile(pattern, flags)
def __call__(self, value):
if value is None:
return None
- value = unicode(value)
+ value = six.text_type(value)
if self.pattern.match(value) is None:
raise ValueError('Expected {}, not {}'.format(self.name, json_encode_string(value)))
return value
def format(self, value):
- return None if value is None else unicode(value)
+ return None if value is None else six.text_type(value)
class OptionName(Validator):
@@ -336,13 +346,13 @@ class OptionName(Validator):
def __call__(self, value):
if value is not None:
- value = unicode(value)
+ value = six.text_type(value)
if OptionName.pattern.match(value) is None:
raise ValueError('Illegal characters in option name: {}'.format(value))
return value
def format(self, value):
- return None if value is None else unicode(value)
+ return None if value is None else six.text_type(value)
class RegularExpression(Validator):
@@ -353,9 +363,9 @@ class RegularExpression(Validator):
if value is None:
return None
try:
- value = re.compile(unicode(value))
+ value = re.compile(six.text_type(value))
except re.error as error:
- raise ValueError('{}: {}'.format(unicode(error).capitalize(), value))
+ raise ValueError('{}: {}'.format(six.text_type(error).capitalize(), value))
return value
def format(self, value):
@@ -372,7 +382,7 @@ class Set(Validator):
def __call__(self, value):
if value is None:
return None
- value = unicode(value)
+ value = six.text_type(value)
if value not in self.membership:
raise ValueError('Unrecognized value: {}'.format(value))
return value
diff --git a/dist/escu/bin/splunklib/six.py b/dist/escu/bin/splunklib/six.py
new file mode 100644
index 0000000000..5fe9f8e141
--- /dev/null
+++ b/dist/escu/bin/splunklib/six.py
@@ -0,0 +1,980 @@
+# Copyright (c) 2010-2020 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+from __future__ import absolute_import
+
+import functools
+import itertools
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson "
+__version__ = "1.14.0"
+
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+PY34 = sys.version_info[0:2] >= (3, 4)
+
+if PY3:
+ string_types = str,
+ integer_types = int,
+ class_types = type,
+ text_type = str
+ binary_type = bytes
+
+ MAXSIZE = sys.maxsize
+else:
+ string_types = basestring,
+ integer_types = (int, long)
+ class_types = (type, types.ClassType)
+ text_type = unicode
+ binary_type = str
+
+ if sys.platform.startswith("java"):
+ # Jython always uses 32 bits.
+ MAXSIZE = int((1 << 31) - 1)
+ else:
+ # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+ class X(object):
+
+ def __len__(self):
+ return 1 << 31
+ try:
+ len(X())
+ except OverflowError:
+ # 32-bit
+ MAXSIZE = int((1 << 31) - 1)
+ else:
+ # 64-bit
+ MAXSIZE = int((1 << 63) - 1)
+ del X
+
+
+def _add_doc(func, doc):
+ """Add documentation to a function."""
+ func.__doc__ = doc
+
+
+def _import_module(name):
+ """Import module, returning the module after the last dot."""
+ __import__(name)
+ return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+ def __init__(self, name):
+ self.name = name
+
+ def __get__(self, obj, tp):
+ result = self._resolve()
+ setattr(obj, self.name, result) # Invokes __set__.
+ try:
+ # This is a bit ugly, but it avoids running this again by
+ # removing this descriptor.
+ delattr(obj.__class__, self.name)
+ except AttributeError:
+ pass
+ return result
+
+
+class MovedModule(_LazyDescr):
+
+ def __init__(self, name, old, new=None):
+ super(MovedModule, self).__init__(name)
+ if PY3:
+ if new is None:
+ new = name
+ self.mod = new
+ else:
+ self.mod = old
+
+ def _resolve(self):
+ return _import_module(self.mod)
+
+ def __getattr__(self, attr):
+ _module = self._resolve()
+ value = getattr(_module, attr)
+ setattr(self, attr, value)
+ return value
+
+
+class _LazyModule(types.ModuleType):
+
+ def __init__(self, name):
+ super(_LazyModule, self).__init__(name)
+ self.__doc__ = self.__class__.__doc__
+
+ def __dir__(self):
+ attrs = ["__doc__", "__name__"]
+ attrs += [attr.name for attr in self._moved_attributes]
+ return attrs
+
+ # Subclasses should override this
+ _moved_attributes = []
+
+
+class MovedAttribute(_LazyDescr):
+
+ def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+ super(MovedAttribute, self).__init__(name)
+ if PY3:
+ if new_mod is None:
+ new_mod = name
+ self.mod = new_mod
+ if new_attr is None:
+ if old_attr is None:
+ new_attr = name
+ else:
+ new_attr = old_attr
+ self.attr = new_attr
+ else:
+ self.mod = old_mod
+ if old_attr is None:
+ old_attr = name
+ self.attr = old_attr
+
+ def _resolve(self):
+ module = _import_module(self.mod)
+ return getattr(module, self.attr)
+
+
+class _SixMetaPathImporter(object):
+
+ """
+ A meta path importer to import six.moves and its submodules.
+
+ This class implements a PEP302 finder and loader. It should be compatible
+ with Python 2.5 and all existing versions of Python3
+ """
+
+ def __init__(self, six_module_name):
+ self.name = six_module_name
+ self.known_modules = {}
+
+ def _add_module(self, mod, *fullnames):
+ for fullname in fullnames:
+ self.known_modules[self.name + "." + fullname] = mod
+
+ def _get_module(self, fullname):
+ return self.known_modules[self.name + "." + fullname]
+
+ def find_module(self, fullname, path=None):
+ if fullname in self.known_modules:
+ return self
+ return None
+
+ def __get_module(self, fullname):
+ try:
+ return self.known_modules[fullname]
+ except KeyError:
+ raise ImportError("This loader does not know module " + fullname)
+
+ def load_module(self, fullname):
+ try:
+ # in case of a reload
+ return sys.modules[fullname]
+ except KeyError:
+ pass
+ mod = self.__get_module(fullname)
+ if isinstance(mod, MovedModule):
+ mod = mod._resolve()
+ else:
+ mod.__loader__ = self
+ sys.modules[fullname] = mod
+ return mod
+
+ def is_package(self, fullname):
+ """
+ Return true, if the named module is a package.
+
+ We need this method to get correct spec objects with
+ Python 3.4 (see PEP451)
+ """
+ return hasattr(self.__get_module(fullname), "__path__")
+
+ def get_code(self, fullname):
+ """Return None
+
+ Required, if is_package is implemented"""
+ self.__get_module(fullname) # eventually raises ImportError
+ return None
+ get_source = get_code # same as get_code
+
+_importer = _SixMetaPathImporter(__name__)
+
+
+class _MovedItems(_LazyModule):
+
+ """Lazy loading of moved objects"""
+ __path__ = [] # mark as package
+
+
+_moved_attributes = [
+ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+ MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+ MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+ MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+ MovedAttribute("intern", "__builtin__", "sys"),
+ MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+ MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
+ MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
+ MovedAttribute("getoutput", "commands", "subprocess"),
+ MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
+ MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
+ MovedAttribute("reduce", "__builtin__", "functools"),
+ MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
+ MovedAttribute("StringIO", "StringIO", "io"),
+ MovedAttribute("UserDict", "UserDict", "collections"),
+ MovedAttribute("UserList", "UserList", "collections"),
+ MovedAttribute("UserString", "UserString", "collections"),
+ MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+ MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+ MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+ MovedModule("builtins", "__builtin__"),
+ MovedModule("configparser", "ConfigParser"),
+ MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"),
+ MovedModule("copyreg", "copy_reg"),
+ MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
+ MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"),
+ MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"),
+ MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+ MovedModule("http_cookies", "Cookie", "http.cookies"),
+ MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+ MovedModule("html_parser", "HTMLParser", "html.parser"),
+ MovedModule("http_client", "httplib", "http.client"),
+ MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+ MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"),
+ MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+ MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
+ MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+ MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+ MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+ MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+ MovedModule("cPickle", "cPickle", "pickle"),
+ MovedModule("queue", "Queue"),
+ MovedModule("reprlib", "repr"),
+ MovedModule("socketserver", "SocketServer"),
+ MovedModule("_thread", "thread", "_thread"),
+ MovedModule("tkinter", "Tkinter"),
+ MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+ MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+ MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+ MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+ MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+ MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
+ MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+ MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+ MovedModule("tkinter_colorchooser", "tkColorChooser",
+ "tkinter.colorchooser"),
+ MovedModule("tkinter_commondialog", "tkCommonDialog",
+ "tkinter.commondialog"),
+ MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+ MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+ MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+ MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+ "tkinter.simpledialog"),
+ MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
+ MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
+ MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
+ MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+ MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
+ MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
+]
+# Add windows specific modules.
+if sys.platform == "win32":
+ _moved_attributes += [
+ MovedModule("winreg", "_winreg"),
+ ]
+
+for attr in _moved_attributes:
+ setattr(_MovedItems, attr.name, attr)
+ if isinstance(attr, MovedModule):
+ _importer._add_module(attr, "moves." + attr.name)
+del attr
+
+_MovedItems._moved_attributes = _moved_attributes
+
+moves = _MovedItems(__name__ + ".moves")
+_importer._add_module(moves, "moves")
+
+
+class Module_six_moves_urllib_parse(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_parse"""
+
+
+_urllib_parse_moved_attributes = [
+ MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
+ MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
+ MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
+ MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
+ MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
+ MovedAttribute("urljoin", "urlparse", "urllib.parse"),
+ MovedAttribute("urlparse", "urlparse", "urllib.parse"),
+ MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
+ MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
+ MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
+ MovedAttribute("quote", "urllib", "urllib.parse"),
+ MovedAttribute("quote_plus", "urllib", "urllib.parse"),
+ MovedAttribute("unquote", "urllib", "urllib.parse"),
+ MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+ MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"),
+ MovedAttribute("urlencode", "urllib", "urllib.parse"),
+ MovedAttribute("splitquery", "urllib", "urllib.parse"),
+ MovedAttribute("splittag", "urllib", "urllib.parse"),
+ MovedAttribute("splituser", "urllib", "urllib.parse"),
+ MovedAttribute("splitvalue", "urllib", "urllib.parse"),
+ MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
+ MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
+ MovedAttribute("uses_params", "urlparse", "urllib.parse"),
+ MovedAttribute("uses_query", "urlparse", "urllib.parse"),
+ MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
+]
+for attr in _urllib_parse_moved_attributes:
+ setattr(Module_six_moves_urllib_parse, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+ "moves.urllib_parse", "moves.urllib.parse")
+
+
+class Module_six_moves_urllib_error(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_error"""
+
+
+_urllib_error_moved_attributes = [
+ MovedAttribute("URLError", "urllib2", "urllib.error"),
+ MovedAttribute("HTTPError", "urllib2", "urllib.error"),
+ MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
+]
+for attr in _urllib_error_moved_attributes:
+ setattr(Module_six_moves_urllib_error, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+ "moves.urllib_error", "moves.urllib.error")
+
+
+class Module_six_moves_urllib_request(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_request"""
+
+
+_urllib_request_moved_attributes = [
+ MovedAttribute("urlopen", "urllib2", "urllib.request"),
+ MovedAttribute("install_opener", "urllib2", "urllib.request"),
+ MovedAttribute("build_opener", "urllib2", "urllib.request"),
+ MovedAttribute("pathname2url", "urllib", "urllib.request"),
+ MovedAttribute("url2pathname", "urllib", "urllib.request"),
+ MovedAttribute("getproxies", "urllib", "urllib.request"),
+ MovedAttribute("Request", "urllib2", "urllib.request"),
+ MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
+ MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
+ MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
+ MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
+ MovedAttribute("FileHandler", "urllib2", "urllib.request"),
+ MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
+ MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
+ MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
+ MovedAttribute("urlretrieve", "urllib", "urllib.request"),
+ MovedAttribute("urlcleanup", "urllib", "urllib.request"),
+ MovedAttribute("URLopener", "urllib", "urllib.request"),
+ MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
+ MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+ MovedAttribute("parse_http_list", "urllib2", "urllib.request"),
+ MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"),
+]
+for attr in _urllib_request_moved_attributes:
+ setattr(Module_six_moves_urllib_request, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+ "moves.urllib_request", "moves.urllib.request")
+
+
+class Module_six_moves_urllib_response(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_response"""
+
+
+_urllib_response_moved_attributes = [
+ MovedAttribute("addbase", "urllib", "urllib.response"),
+ MovedAttribute("addclosehook", "urllib", "urllib.response"),
+ MovedAttribute("addinfo", "urllib", "urllib.response"),
+ MovedAttribute("addinfourl", "urllib", "urllib.response"),
+]
+for attr in _urllib_response_moved_attributes:
+ setattr(Module_six_moves_urllib_response, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+ "moves.urllib_response", "moves.urllib.response")
+
+
+class Module_six_moves_urllib_robotparser(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_robotparser"""
+
+
+_urllib_robotparser_moved_attributes = [
+ MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+]
+for attr in _urllib_robotparser_moved_attributes:
+ setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+ "moves.urllib_robotparser", "moves.urllib.robotparser")
+
+
+class Module_six_moves_urllib(types.ModuleType):
+
+ """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+ __path__ = [] # mark as package
+ parse = _importer._get_module("moves.urllib_parse")
+ error = _importer._get_module("moves.urllib_error")
+ request = _importer._get_module("moves.urllib_request")
+ response = _importer._get_module("moves.urllib_response")
+ robotparser = _importer._get_module("moves.urllib_robotparser")
+
+ def __dir__(self):
+ return ['parse', 'error', 'request', 'response', 'robotparser']
+
+_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
+ "moves.urllib")
+
+
+def add_move(move):
+ """Add an item to six.moves."""
+ setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+ """Remove item from six.moves."""
+ try:
+ delattr(_MovedItems, name)
+ except AttributeError:
+ try:
+ del moves.__dict__[name]
+ except KeyError:
+ raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+ _meth_func = "__func__"
+ _meth_self = "__self__"
+
+ _func_closure = "__closure__"
+ _func_code = "__code__"
+ _func_defaults = "__defaults__"
+ _func_globals = "__globals__"
+else:
+ _meth_func = "im_func"
+ _meth_self = "im_self"
+
+ _func_closure = "func_closure"
+ _func_code = "func_code"
+ _func_defaults = "func_defaults"
+ _func_globals = "func_globals"
+
+
+try:
+ advance_iterator = next
+except NameError:
+ def advance_iterator(it):
+ return it.next()
+next = advance_iterator
+
+
+try:
+ callable = callable
+except NameError:
+ def callable(obj):
+ return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+ def get_unbound_function(unbound):
+ return unbound
+
+ create_bound_method = types.MethodType
+
+ def create_unbound_method(func, cls):
+ return func
+
+ Iterator = object
+else:
+ def get_unbound_function(unbound):
+ return unbound.im_func
+
+ def create_bound_method(func, obj):
+ return types.MethodType(func, obj, obj.__class__)
+
+ def create_unbound_method(func, cls):
+ return types.MethodType(func, None, cls)
+
+ class Iterator(object):
+
+ def next(self):
+ return type(self).__next__(self)
+
+ callable = callable
+_add_doc(get_unbound_function,
+ """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_closure = operator.attrgetter(_func_closure)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+get_function_globals = operator.attrgetter(_func_globals)
+
+
+if PY3:
+ def iterkeys(d, **kw):
+ return iter(d.keys(**kw))
+
+ def itervalues(d, **kw):
+ return iter(d.values(**kw))
+
+ def iteritems(d, **kw):
+ return iter(d.items(**kw))
+
+ def iterlists(d, **kw):
+ return iter(d.lists(**kw))
+
+ viewkeys = operator.methodcaller("keys")
+
+ viewvalues = operator.methodcaller("values")
+
+ viewitems = operator.methodcaller("items")
+else:
+ def iterkeys(d, **kw):
+ return d.iterkeys(**kw)
+
+ def itervalues(d, **kw):
+ return d.itervalues(**kw)
+
+ def iteritems(d, **kw):
+ return d.iteritems(**kw)
+
+ def iterlists(d, **kw):
+ return d.iterlists(**kw)
+
+ viewkeys = operator.methodcaller("viewkeys")
+
+ viewvalues = operator.methodcaller("viewvalues")
+
+ viewitems = operator.methodcaller("viewitems")
+
+_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
+_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
+_add_doc(iteritems,
+ "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(iterlists,
+ "Return an iterator over the (key, [values]) pairs of a dictionary.")
+
+
+if PY3:
+ def b(s):
+ return s.encode("latin-1")
+
+ def u(s):
+ return s
+ unichr = chr
+ import struct
+ int2byte = struct.Struct(">B").pack
+ del struct
+ byte2int = operator.itemgetter(0)
+ indexbytes = operator.getitem
+ iterbytes = iter
+ import io
+ StringIO = io.StringIO
+ BytesIO = io.BytesIO
+ del io
+ _assertCountEqual = "assertCountEqual"
+ if sys.version_info[1] <= 1:
+ _assertRaisesRegex = "assertRaisesRegexp"
+ _assertRegex = "assertRegexpMatches"
+ _assertNotRegex = "assertNotRegexpMatches"
+ else:
+ _assertRaisesRegex = "assertRaisesRegex"
+ _assertRegex = "assertRegex"
+ _assertNotRegex = "assertNotRegex"
+else:
+ def b(s):
+ return s
+ # Workaround for standalone backslash
+
+ def u(s):
+ return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+ unichr = unichr
+ int2byte = chr
+
+ def byte2int(bs):
+ return ord(bs[0])
+
+ def indexbytes(buf, i):
+ return ord(buf[i])
+ iterbytes = functools.partial(itertools.imap, ord)
+ import StringIO
+ StringIO = BytesIO = StringIO.StringIO
+ _assertCountEqual = "assertItemsEqual"
+ _assertRaisesRegex = "assertRaisesRegexp"
+ _assertRegex = "assertRegexpMatches"
+ _assertNotRegex = "assertNotRegexpMatches"
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+def assertCountEqual(self, *args, **kwargs):
+ return getattr(self, _assertCountEqual)(*args, **kwargs)
+
+
+def assertRaisesRegex(self, *args, **kwargs):
+ return getattr(self, _assertRaisesRegex)(*args, **kwargs)
+
+
+def assertRegex(self, *args, **kwargs):
+ return getattr(self, _assertRegex)(*args, **kwargs)
+
+
+def assertNotRegex(self, *args, **kwargs):
+ return getattr(self, _assertNotRegex)(*args, **kwargs)
+
+
+if PY3:
+ exec_ = getattr(moves.builtins, "exec")
+
+ def reraise(tp, value, tb=None):
+ try:
+ if value is None:
+ value = tp()
+ if value.__traceback__ is not tb:
+ raise value.with_traceback(tb)
+ raise value
+ finally:
+ value = None
+ tb = None
+
+else:
+ def exec_(_code_, _globs_=None, _locs_=None):
+ """Execute code in a namespace."""
+ if _globs_ is None:
+ frame = sys._getframe(1)
+ _globs_ = frame.f_globals
+ if _locs_ is None:
+ _locs_ = frame.f_locals
+ del frame
+ elif _locs_ is None:
+ _locs_ = _globs_
+ exec("""exec _code_ in _globs_, _locs_""")
+
+ exec_("""def reraise(tp, value, tb=None):
+ try:
+ raise tp, value, tb
+ finally:
+ tb = None
+""")
+
+
+if sys.version_info[:2] > (3,):
+ exec_("""def raise_from(value, from_value):
+ try:
+ raise value from from_value
+ finally:
+ value = None
+""")
+else:
+ def raise_from(value, from_value):
+ raise value
+
+
+print_ = getattr(moves.builtins, "print", None)
+if print_ is None:
+ def print_(*args, **kwargs):
+ """The new-style print function for Python 2.4 and 2.5."""
+ fp = kwargs.pop("file", sys.stdout)
+ if fp is None:
+ return
+
+ def write(data):
+ if not isinstance(data, basestring):
+ data = str(data)
+ # If the file has an encoding, encode unicode with it.
+ if (isinstance(fp, file) and
+ isinstance(data, unicode) and
+ fp.encoding is not None):
+ errors = getattr(fp, "errors", None)
+ if errors is None:
+ errors = "strict"
+ data = data.encode(fp.encoding, errors)
+ fp.write(data)
+ want_unicode = False
+ sep = kwargs.pop("sep", None)
+ if sep is not None:
+ if isinstance(sep, unicode):
+ want_unicode = True
+ elif not isinstance(sep, str):
+ raise TypeError("sep must be None or a string")
+ end = kwargs.pop("end", None)
+ if end is not None:
+ if isinstance(end, unicode):
+ want_unicode = True
+ elif not isinstance(end, str):
+ raise TypeError("end must be None or a string")
+ if kwargs:
+ raise TypeError("invalid keyword arguments to print()")
+ if not want_unicode:
+ for arg in args:
+ if isinstance(arg, unicode):
+ want_unicode = True
+ break
+ if want_unicode:
+ newline = unicode("\n")
+ space = unicode(" ")
+ else:
+ newline = "\n"
+ space = " "
+ if sep is None:
+ sep = space
+ if end is None:
+ end = newline
+ for i, arg in enumerate(args):
+ if i:
+ write(sep)
+ write(arg)
+ write(end)
+if sys.version_info[:2] < (3, 3):
+ _print = print_
+
+ def print_(*args, **kwargs):
+ fp = kwargs.get("file", sys.stdout)
+ flush = kwargs.pop("flush", False)
+ _print(*args, **kwargs)
+ if flush and fp is not None:
+ fp.flush()
+
+_add_doc(reraise, """Reraise an exception.""")
+
+if sys.version_info[0:2] < (3, 4):
+ # This does exactly the same what the :func:`py3:functools.update_wrapper`
+ # function does on Python versions after 3.2. It sets the ``__wrapped__``
+ # attribute on ``wrapper`` object and it doesn't raise an error if any of
+ # the attributes mentioned in ``assigned`` and ``updated`` are missing on
+ # ``wrapped`` object.
+ def _update_wrapper(wrapper, wrapped,
+ assigned=functools.WRAPPER_ASSIGNMENTS,
+ updated=functools.WRAPPER_UPDATES):
+ for attr in assigned:
+ try:
+ value = getattr(wrapped, attr)
+ except AttributeError:
+ continue
+ else:
+ setattr(wrapper, attr, value)
+ for attr in updated:
+ getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
+ wrapper.__wrapped__ = wrapped
+ return wrapper
+ _update_wrapper.__doc__ = functools.update_wrapper.__doc__
+
+ def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
+ updated=functools.WRAPPER_UPDATES):
+ return functools.partial(_update_wrapper, wrapped=wrapped,
+ assigned=assigned, updated=updated)
+ wraps.__doc__ = functools.wraps.__doc__
+
+else:
+ wraps = functools.wraps
+
+
+def with_metaclass(meta, *bases):
+ """Create a base class with a metaclass."""
+ # This requires a bit of explanation: the basic idea is to make a dummy
+ # metaclass for one level of class instantiation that replaces itself with
+ # the actual metaclass.
+ class metaclass(type):
+
+ def __new__(cls, name, this_bases, d):
+ if sys.version_info[:2] >= (3, 7):
+ # This version introduced PEP 560 that requires a bit
+ # of extra care (we mimic what is done by __build_class__).
+ resolved_bases = types.resolve_bases(bases)
+ if resolved_bases is not bases:
+ d['__orig_bases__'] = bases
+ else:
+ resolved_bases = bases
+ return meta(name, resolved_bases, d)
+
+ @classmethod
+ def __prepare__(cls, name, this_bases):
+ return meta.__prepare__(name, bases)
+ return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+def add_metaclass(metaclass):
+ """Class decorator for creating a class with a metaclass."""
+ def wrapper(cls):
+ orig_vars = cls.__dict__.copy()
+ slots = orig_vars.get('__slots__')
+ if slots is not None:
+ if isinstance(slots, str):
+ slots = [slots]
+ for slots_var in slots:
+ orig_vars.pop(slots_var)
+ orig_vars.pop('__dict__', None)
+ orig_vars.pop('__weakref__', None)
+ if hasattr(cls, '__qualname__'):
+ orig_vars['__qualname__'] = cls.__qualname__
+ return metaclass(cls.__name__, cls.__bases__, orig_vars)
+ return wrapper
+
+
+def ensure_binary(s, encoding='utf-8', errors='strict'):
+ """Coerce **s** to six.binary_type.
+
+ For Python 2:
+ - `unicode` -> encoded to `str`
+ - `str` -> `str`
+
+ For Python 3:
+ - `str` -> encoded to `bytes`
+ - `bytes` -> `bytes`
+ """
+ if isinstance(s, text_type):
+ return s.encode(encoding, errors)
+ elif isinstance(s, binary_type):
+ return s
+ else:
+ raise TypeError("not expecting type '%s'" % type(s))
+
+
+def ensure_str(s, encoding='utf-8', errors='strict'):
+ """Coerce *s* to `str`.
+
+ For Python 2:
+ - `unicode` -> encoded to `str`
+ - `str` -> `str`
+
+ For Python 3:
+ - `str` -> `str`
+ - `bytes` -> decoded to `str`
+ """
+ if not isinstance(s, (text_type, binary_type)):
+ raise TypeError("not expecting type '%s'" % type(s))
+ if PY2 and isinstance(s, text_type):
+ s = s.encode(encoding, errors)
+ elif PY3 and isinstance(s, binary_type):
+ s = s.decode(encoding, errors)
+ return s
+
+
+def ensure_text(s, encoding='utf-8', errors='strict'):
+ """Coerce *s* to six.text_type.
+
+ For Python 2:
+ - `unicode` -> `unicode`
+ - `str` -> `unicode`
+
+ For Python 3:
+ - `str` -> `str`
+ - `bytes` -> decoded to `str`
+ """
+ if isinstance(s, binary_type):
+ return s.decode(encoding, errors)
+ elif isinstance(s, text_type):
+ return s
+ else:
+ raise TypeError("not expecting type '%s'" % type(s))
+
+
+def python_2_unicode_compatible(klass):
+ """
+ A class decorator that defines __unicode__ and __str__ methods under Python 2.
+ Under Python 3 it does nothing.
+
+ To support Python 2 and 3 with a single code base, define a __str__ method
+ returning text and apply this decorator to the class.
+ """
+ if PY2:
+ if '__str__' not in klass.__dict__:
+ raise ValueError("@python_2_unicode_compatible cannot be applied "
+ "to %s because it doesn't define __str__()." %
+ klass.__name__)
+ klass.__unicode__ = klass.__str__
+ klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+ return klass
+
+
+# Complete the moves implementation.
+# This code is at the end of this module to speed up module loading.
+# Turn this module into a package.
+__path__ = [] # required for PEP 302 and PEP 451
+__package__ = __name__ # see PEP 366 @ReservedAssignment
+if globals().get("__spec__") is not None:
+ __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable
+# Remove other six meta path importers, since they cause problems. This can
+# happen if six is removed from sys.modules and then reloaded. (Setuptools does
+# this for some reason.)
+if sys.meta_path:
+ for i, importer in enumerate(sys.meta_path):
+ # Here's some real nastiness: Another "instance" of the six module might
+ # be floating around. Therefore, we can't use isinstance() to check for
+ # the six meta path importer, since the other six instance will have
+ # inserted an importer with different class.
+ if (type(importer).__name__ == "_SixMetaPathImporter" and
+ importer.name == __name__):
+ del sys.meta_path[i]
+ break
+ del i, importer
+# Finally, add the importer to the meta path import hook.
+sys.meta_path.append(_importer)