<commit_before># xpyBuild - eXtensible Python-based Build System
#
# This module holds definitions that are used throughout the build system, and
# typically all names from this module will be imported.
#
# Copyright (c) 2013 - 2017, 2019 Software AG, Darmstadt, Germany and/or its licensors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# $Id: buildcommon.py 301527 2017-02-06 15:31:43Z matj $
#

"""
Contains standard functionality for use in build files such as `xpybuild.buildcommon.include`, useful constants such as `xpybuild.buildcommon.IS_WINDOWS` and
functionality for adding prefixes/suffixes to paths such as `xpybuild.buildcommon.FilenameStringFormatter`.
"""

import traceback, os, sys, locale, inspect, io
import re
import platform

import logging
# do NOT define a 'log' variable here or targets will use it by mistake

def __getXpybuildVersion():

	try:
		with open(os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "XPYBUILD_VERSION")) as f:
			return f.read().strip()
	except Exception:
		raise
		return "<unknown>"
XPYBUILD_VERSION: str = __getXpybuildVersion()
"""The current xpybuild version."""

def include(file):
	""" Parse and register the targets and properties in in the specified
	``XXX.xpybuild.py`` file.

	Targets should only be defined in files included using this method,
	not using python import statements.

	@param file: a path relative to the directory containing this file.
	"""

	from xpybuild.buildcontext import getBuildInitializationContext
	from xpybuild.utils.buildfilelocation import BuildFileLocation

	file = getBuildInitializationContext().expandPropertyValues(file)

	assert file.endswith('.xpybuild.py') # enforce recommended naming convention

	filepath = getBuildInitializationContext().getFullPath(file, os.path.dirname(BuildFileLocation._currentBuildFile[-1]))

	BuildFileLocation._currentBuildFile.append(filepath) # add to stack of files being parsed

	namespace = {}
	exec(compile(open(filepath, "rb").read(), filepath, 'exec'), namespace, namespace)

	del BuildFileLocation._currentBuildFile[-1]

	return namespace

IS_WINDOWS: bool = platform.system()=='Windows'
""" A boolean that specifies whether this is Windows or some other operating system. """
# (we won't want constants for every possible OS here, but since there is so much conditionalization between
# windows and unix-based systems, much of it on the critical path, it is worthwhile having a constant for this).

if IS_WINDOWS:
	def isWindows():
		""" Returns True if this is a windows platform.

		@deprecated: Use the `IS_WINDOWS` constant instead.
		"""
		return True
else:
	def isWindows():
		""" Returns True if this is a windows platform.

		@deprecated: Use the `IS_WINDOWS` constant instead.
		"""
		return False

PREFERRED_ENCODING = locale.getpreferredencoding()
"""
The operating system's preferred/default encoding for reading/writing the contents of text data in files and
process stdout/stderr for the current environment (or machine).

This returns the same value as Python's ``locale.getpreferredencoding()`` method, but as that method is not thread-safe,
this constant must always be used in test cases to avoid race conditions when running tests in parallel.

The OS preferred encoding should not be confused with Python's 'default' encoding (``sys.getdefaultencoding()``) which
is usually not relevant in a build.

.. versionadded:: 4.0
"""

def defineAtomicTargetGroup(*targets):
	""" The given targets must all be built before anything which depends on any of those targets.

	Returns the flattened list of targets.
	"""

	from xpybuild.buildcontext import getBuildInitializationContext
	targets = flatten(targets)
	getBuildInitializationContext().defineAtomicTargetGroup(targets)
	return targets

def requireXpybuildVersion(version: str):
	""" Checks that this xpybuild is at least a certain version number. """
	from xpybuild.utils.stringutils import compareVersions
	if compareVersions(XPYBUILD_VERSION, version) < 0: raise Exception("This build file requires xpyBuild at least version "+version+" but this is xpyBuild "+XPYBUILD_VERSION)

requireXpyBuildVersion = requireXpybuildVersion
"""
.. private:: Old name for compatibility.

Use requireXpyBuildVersion instead.
"""

def registerBuildLoadPostProcessor(fn):
	""" Add a callback function that will be invoked just after the build files have been parsed and all targets added,
	to allow for custom post processing logic to execute before the clean/build.

	This is useful for performing global customizations of targets throughout the build, such as adding tags or
	additional target options. For example, to override the default retry setting and add a tag to all defined targets
	of a particular Python class::

		def demoPostProcessor(context):
			for target in context.targets().values():
				if isinstance(target, MyCustomTarget) or 'FooBar' in target.name:
					# Override options for these targets
					target.option(BaseTarget.Options.failureRetries, 2)

					# Add additional tags for these targets
					target.tags('my-post-processed-tag')

		registerBuildLoadPostProcessor(demoPostProcessor)

	:param Callable[xpybuild.buildcontext.BuildInitializationContext] fn: A function that takes a context as its argument.
	You may wish to use ``BuildInitializationContext`` methods such as `xpybuild.buildcontext.BuildInitializationContext.targets` from your function.
	"""
	from xpybuild.buildcontext import getBuildInitializationContext
	if not fn: return
	getBuildInitializationContext()._buildParsePostProcessors.append(fn)

def registerPreBuildCheck(fn):
	""" Defines a check which will be called after any clean but before any build actions take place, to provide fail-fast
	behaviour if something is wrong.

	See also `registerBuildLoadPostProcessor`.

	:param Callable[xpybuild.buildcontext.BuildInitializationContext] fn: A function that takes a context as its argument,
	and raises a BuildException if the check fails.

	"""
	from xpybuild.buildcontext import getBuildInitializationContext
	getBuildInitializationContext().registerPreBuildCheck(fn)

class StringFormatter(object):
	""" A simple named functor for applying a ``%s``-style string format, useful
	in situations where a function is needed to add a suffix/prefix for the
	value of an option.
	"""
	def __init__(self, formatstring):
		self.fmt = formatstring
	def __repr__(self):
		return 'StringFormatter<"%s">'%self.fmt
	def __call__(self, *args, **kwargs):
		assert not kwargs
		assert len(args)==1
		return self.fmt % args[0]

class FilenameStringFormatter(object):
	""" A simple named functor for applying a ``%s``-style string format.
		 Formatter is just applied to the basename part of the filename,
		 the dirname part is preserved as-is.
	"""
	def __init__(self, formatstring):
		self.fmt = formatstring
	def __repr__(self):
		return 'FilenameStringFormatter<"%s">'%self.fmt
	def __call__(self, *args, **kwargs):
		assert not kwargs
		assert len(args)==1
		return os.path.join(os.path.dirname(args[0]), self.fmt % os.path.basename(args[0]))

import pkgutil

def enableLegacyXpybuildModuleNames():
	"""
	Adds aliases for pre-3.0 module names e.g. `buildcommon` instead of `xpybuild.buildcommon`, etc.

	The old names are deprecated, so this should be used only as a temporary measure.
	"""
	# must manually import every single module from 'utils' and 'targets';
	# although just importing 'utils' appears to work,
	# we end up with duplicate packages for the modules underneath it
	# (and so may attempt to import and hence define the same option twice)
	import xpybuild.utils
	import xpybuild.targets
	__log = logging.getLogger('xpybuild.buildcommon')
	for parentpackage in [xpybuild, xpybuild.utils, xpybuild.targets]:
		for _, modulename, ispkg in pkgutil.iter_modules(
				path=parentpackage.__path__, prefix=(parentpackage.__name__[len('xpybuild.'):]+'.').lstrip('.')):
			__log.debug('enableLegacyXpybuildModuleNames: Importing legacy package name %s', modulename)
			if modulename!='buildcommon': # first make sure the original one has been imported
				exec(f'import xpybuild.{modulename}', {})
			# then define an alias
			exec(f'sys.modules["{modulename}"] = sys.modules["xpybuild.{modulename}"]')
	assert 'utils.fileutils' in sys.modules, sys.modules # sanity check that it worked
	assert 'targets.copy' in sys.modules, sys.modules # sanity check that it worked

	# aliases for modules we folded into other modules in v3.0, or moved
	exec(f'sys.modules["propertyfunctors"] = sys.modules["xpybuild.propertysupport"]')
	exec(f'sys.modules["buildexceptions"] = sys.modules["xpybuild.utils.buildexceptions"]')

	xpybuild.targets.touch = sys.modules["xpybuild.targets.writefile"]
	exec(f'sys.modules["targets.touch"] = sys.modules["xpybuild.targets.writefile"]')

	xpybuild.targets.unpack = sys.modules["xpybuild.targets.archive"]
	exec(f'sys.modules["targets.unpack"] = sys.modules["xpybuild.targets.archive"]')
	xpybuild.targets.zip = sys.modules["xpybuild.targets.archive"]
	exec(f'sys.modules["targets.zip"] = sys.modules["xpybuild.targets.archive"]')
	xpybuild.targets.tar = sys.modules["xpybuild.targets.archive"]
	exec(f'sys.modules["targets.tar"] = sys.modules["xpybuild.targets.archive"]')

import xpybuild.utils.fileutils
from xpybuild.utils.flatten import flatten

isDirPath = xpybuild.utils.fileutils.isDirPath
"""Returns true if the path is a directory (ends with a slash, ``/`` or ``\\\\``). """

normpath = xpybuild.utils.fileutils.normPath
"""
.. private:: This is deprecated in favour of fileutils.normPath and hidden from documentation to avoid polluting the docs.

"""
<commit_msg>fix another typo in PREFERED_ENCODING doc<commit_after># xpyBuild - eXtensible Python-based Build System
#
# This module holds definitions that are used throughout the build system, and
# typically all names from this module will be imported.
#
# Copyright (c) 2013 - 2017, 2019 Software AG, Darmstadt, Germany and/or its licensors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# $Id: buildcommon.py 301527 2017-02-06 15:31:43Z matj $
#

"""
Contains standard functionality for use in build files such as `xpybuild.buildcommon.include`, useful constants such as `xpybuild.buildcommon.IS_WINDOWS` and
functionality for adding prefixes/suffixes to paths such as `xpybuild.buildcommon.FilenameStringFormatter`.
"""

import traceback, os, sys, locale, inspect, io
import re
import platform

import logging
# do NOT define a 'log' variable here or targets will use it by mistake

def __getXpybuildVersion():

	try:
		with open(os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "XPYBUILD_VERSION")) as f:
			return f.read().strip()
	except Exception:
		raise
		return "<unknown>"
XPYBUILD_VERSION: str = __getXpybuildVersion()
"""The current xpybuild version."""

def include(file):
	""" Parse and register the targets and properties in in the specified
	``XXX.xpybuild.py`` file.

	Targets should only be defined in files included using this method,
	not using python import statements.

	@param file: a path relative to the directory containing this file.
	"""

	from xpybuild.buildcontext import getBuildInitializationContext
	from xpybuild.utils.buildfilelocation import BuildFileLocation

	file = getBuildInitializationContext().expandPropertyValues(file)

	assert file.endswith('.xpybuild.py') # enforce recommended naming convention

	filepath = getBuildInitializationContext().getFullPath(file, os.path.dirname(BuildFileLocation._currentBuildFile[-1]))

	BuildFileLocation._currentBuildFile.append(filepath) # add to stack of files being parsed

	namespace = {}
	exec(compile(open(filepath, "rb").read(), filepath, 'exec'), namespace, namespace)

	del BuildFileLocation._currentBuildFile[-1]

	return namespace

IS_WINDOWS: bool = platform.system()=='Windows'
""" A boolean that specifies whether this is Windows or some other operating system. """
# (we won't want constants for every possible OS here, but since there is so much conditionalization between
# windows and unix-based systems, much of it on the critical path, it is worthwhile having a constant for this).

if IS_WINDOWS:
	def isWindows():
		""" Returns True if this is a windows platform.

		@deprecated: Use the `IS_WINDOWS` constant instead.
		"""
		return True
else:
	def isWindows():
		""" Returns True if this is a windows platform.

		@deprecated: Use the `IS_WINDOWS` constant instead.
		"""
		return False

PREFERRED_ENCODING = locale.getpreferredencoding()
"""
The operating system's preferred/default encoding for reading/writing the contents of text data in files and
process stdout/stderr for the current environment (or machine).

This returns the same value as Python's ``locale.getpreferredencoding()`` method, but as that method is not thread-safe,
this constant must always be used in builds to avoid race conditions when running in parallel.

The OS preferred encoding should not be confused with Python's 'default' encoding (``sys.getdefaultencoding()``) which
is usually not relevant in a build.

.. versionadded:: 4.0
"""

def defineAtomicTargetGroup(*targets):
	""" The given targets must all be built before anything which depends on any of those targets.

	Returns the flattened list of targets.
	"""

	from xpybuild.buildcontext import getBuildInitializationContext
	targets = flatten(targets)
	getBuildInitializationContext().defineAtomicTargetGroup(targets)
	return targets

def requireXpybuildVersion(version: str):
	""" Checks that this xpybuild is at least a certain version number. """
	from xpybuild.utils.stringutils import compareVersions
	if compareVersions(XPYBUILD_VERSION, version) < 0: raise Exception("This build file requires xpyBuild at least version "+version+" but this is xpyBuild "+XPYBUILD_VERSION)

requireXpyBuildVersion = requireXpybuildVersion
"""
.. private:: Old name for compatibility.

Use requireXpyBuildVersion instead.
"""

def registerBuildLoadPostProcessor(fn):
	""" Add a callback function that will be invoked just after the build files have been parsed and all targets added,
	to allow for custom post processing logic to execute before the clean/build.

	This is useful for performing global customizations of targets throughout the build, such as adding tags or
	additional target options. For example, to override the default retry setting and add a tag to all defined targets
	of a particular Python class::

		def demoPostProcessor(context):
			for target in context.targets().values():
				if isinstance(target, MyCustomTarget) or 'FooBar' in target.name:
					# Override options for these targets
					target.option(BaseTarget.Options.failureRetries, 2)

					# Add additional tags for these targets
					target.tags('my-post-processed-tag')

		registerBuildLoadPostProcessor(demoPostProcessor)

	:param Callable[xpybuild.buildcontext.BuildInitializationContext] fn: A function that takes a context as its argument.
	You may wish to use ``BuildInitializationContext`` methods such as `xpybuild.buildcontext.BuildInitializationContext.targets` from your function.
	"""
	from xpybuild.buildcontext import getBuildInitializationContext
	if not fn: return
	getBuildInitializationContext()._buildParsePostProcessors.append(fn)

def registerPreBuildCheck(fn):
	""" Defines a check which will be called after any clean but before any build actions take place, to provide fail-fast
	behaviour if something is wrong.

	See also `registerBuildLoadPostProcessor`.

	:param Callable[xpybuild.buildcontext.BuildInitializationContext] fn: A function that takes a context as its argument,
	and raises a BuildException if the check fails.

	"""
	from xpybuild.buildcontext import getBuildInitializationContext
	getBuildInitializationContext().registerPreBuildCheck(fn)

class StringFormatter(object):
	""" A simple named functor for applying a ``%s``-style string format, useful
	in situations where a function is needed to add a suffix/prefix for the
	value of an option.
	"""
	def __init__(self, formatstring):
		self.fmt = formatstring
	def __repr__(self):
		return 'StringFormatter<"%s">'%self.fmt
	def __call__(self, *args, **kwargs):
		assert not kwargs
		assert len(args)==1
		return self.fmt % args[0]

class FilenameStringFormatter(object):
	""" A simple named functor for applying a ``%s``-style string format.
		 Formatter is just applied to the basename part of the filename,
		 the dirname part is preserved as-is.
	"""
	def __init__(self, formatstring):
		self.fmt = formatstring
	def __repr__(self):
		return 'FilenameStringFormatter<"%s">'%self.fmt
	def __call__(self, *args, **kwargs):
		assert not kwargs
		assert len(args)==1
		return os.path.join(os.path.dirname(args[0]), self.fmt % os.path.basename(args[0]))

import pkgutil

def enableLegacyXpybuildModuleNames():
	"""
	Adds aliases for pre-3.0 module names e.g. `buildcommon` instead of `xpybuild.buildcommon`, etc.

	The old names are deprecated, so this should be used only as a temporary measure.
	"""
	# must manually import every single module from 'utils' and 'targets';
	# although just importing 'utils' appears to work,
	# we end up with duplicate packages for the modules underneath it
	# (and so may attempt to import and hence define the same option twice)
	import xpybuild.utils
	import xpybuild.targets
	__log = logging.getLogger('xpybuild.buildcommon')
	for parentpackage in [xpybuild, xpybuild.utils, xpybuild.targets]:
		for _, modulename, ispkg in pkgutil.iter_modules(
				path=parentpackage.__path__, prefix=(parentpackage.__name__[len('xpybuild.'):]+'.').lstrip('.')):
			__log.debug('enableLegacyXpybuildModuleNames: Importing legacy package name %s', modulename)
			if modulename!='buildcommon': # first make sure the original one has been imported
				exec(f'import xpybuild.{modulename}', {})
			# then define an alias
			exec(f'sys.modules["{modulename}"] = sys.modules["xpybuild.{modulename}"]')
	assert 'utils.fileutils' in sys.modules, sys.modules # sanity check that it worked
	assert 'targets.copy' in sys.modules, sys.modules # sanity check that it worked

	# aliases for modules we folded into other modules in v3.0, or moved
	exec(f'sys.modules["propertyfunctors"] = sys.modules["xpybuild.propertysupport"]')
	exec(f'sys.modules["buildexceptions"] = sys.modules["xpybuild.utils.buildexceptions"]')

	xpybuild.targets.touch = sys.modules["xpybuild.targets.writefile"]
	exec(f'sys.modules["targets.touch"] = sys.modules["xpybuild.targets.writefile"]')

	xpybuild.targets.unpack = sys.modules["xpybuild.targets.archive"]
	exec(f'sys.modules["targets.unpack"] = sys.modules["xpybuild.targets.archive"]')
	xpybuild.targets.zip = sys.modules["xpybuild.targets.archive"]
	exec(f'sys.modules["targets.zip"] = sys.modules["xpybuild.targets.archive"]')
	xpybuild.targets.tar = sys.modules["xpybuild.targets.archive"]
	exec(f'sys.modules["targets.tar"] = sys.modules["xpybuild.targets.archive"]')

import xpybuild.utils.fileutils
from xpybuild.utils.flatten import flatten

isDirPath = xpybuild.utils.fileutils.isDirPath
"""Returns true if the path is a directory (ends with a slash, ``/`` or ``\\\\``). """

normpath = xpybuild.utils.fileutils.normPath
"""
.. private:: This is deprecated in favour of fileutils.normPath and hidden from documentation to avoid polluting the docs.

"""
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
XROXY Parse Lib
v0.4
"""
from grab import Grab

STARTLINK = 'http://www.xroxy.com/proxylist.php?port=&type=All_http&ssl=&country=&latency=&reliability=&sort=reliability&desc=true'

class XroxyLib():
    def __init__(self):
        self.g = Grab()

    def main(self):
        pages = self.get_pages(STARTLINK)
        pg_list = []
        for i in pages:
            pg_list.extend(self.parsepage(i))
        return pg_list

    def print_lst(self, prefix=''):
        newlist = self.main()
        for i in newlist:
            print(prefix + i["ip"] + ":" + i["port"])

    def get_pages(self, link):
        self.g.go(link)
        # Getting total proxy count
        pcount = int(self.g.doc.select('/html/body/div/div[2]/table[2]/tr/td[1]/table/tr[2]/td/small/b').text())
        pagescount = pcount // 10 + 1
        pagelist = []
        for i in range(pagescount):
            pagelink = STARTLINK + '&pnum=' + str(i)
            pagelist.append(pagelink)
        return(pagelist)

    def parsepage(self, link):
        self.g.go(link)
        lst = self.g.doc.select('/html/body/div/div[2]/table[1]/tr[@class]/td')
        if len(lst) < 1:
            return
        return self.parse_lst(lst)

    def parse_lst(self, lst):
        """
        0 - foxyproxy link -- deleting
        1 - ip
        2 - port
        3 - proxy type
        4 - SSL support
        5 - proxy location
        6 - latency
        7 - reliability
        8 - Details -- deleting
        """
        proxylist = []
        it = 1
        proxyd = {}
        for i in lst:
            stri = i.text().strip()
            if len(stri) < 2 or stri == 'Details':
                continue
            if it == 1:
                proxyd["ip"] = stri
            if it == 2:
                proxyd["port"] = stri
            if it == 3:
                proxyd["type"] = stri
            if it == 4:
                proxyd["ssl"] = stri
            if it == 5:
                proxyd["location"] = stri
            if it == 6:
                proxyd["latency"] = stri
            if it == 7:
                proxyd["reliability"] = stri
            it += 1
            if it > 7:
                proxylist.append(proxyd)
                proxyd = {}
                it = 1
        return proxylist

xl = XroxyLib()
xl.print_lst(prefix='http://')
<commit_msg>Empty result fix<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
XROXY Parse Lib
v0.4
"""
from grab import Grab

STARTLINK = 'http://www.xroxy.com/proxylist.php?port=&type=All_http&ssl=&country=&latency=3000&reliability=9000&sort=latency'

class XroxyLib():
    def __init__(self):
        self.g = Grab()

    def main(self):
        pages = self.get_pages(STARTLINK)
        pg_list = []
        for i in pages:
            pg_list.extend(self.parsepage(i))
        return pg_list

    def print_lst(self, prefix=''):
        newlist = self.main()
        for i in newlist:
            print(prefix + i["ip"] + ":" + i["port"])

    def get_pages(self, link):
        self.g.go(link)
        # Getting total proxy count
        pcount = int(self.g.doc.select('/html/body/div/div[2]/table[2]/tr/td[1]/table/tr[2]/td/small/b').text())
        pagescount = pcount // 10 + 1
        pagelist = []
        for i in range(pagescount):
            pagelink = STARTLINK + '&pnum=' + str(i)
            pagelist.append(pagelink)
        return(pagelist)

    def parsepage(self, link):
        self.g.go(link)
        lst = self.g.doc.select('/html/body/div/div[2]/table[1]/tr[@class]/td')
        if len(lst) < 1:
            return []
        return self.parse_lst(lst)

    def parse_lst(self, lst):
        """
        0 - foxyproxy link -- deleting
        1 - ip
        2 - port
        3 - proxy type
        4 - SSL support
        5 - proxy location
        6 - latency
        7 - reliability
        8 - Details -- deleting
        """
        proxylist = []
        it = 1
        proxyd = {}
        for i in lst:
            stri = i.text().strip()
            if len(stri) < 2 or stri == 'Details':
                continue
            if it == 1:
                proxyd["ip"] = stri
            if it == 2:
                proxyd["port"] = stri
            if it == 3:
                proxyd["type"] = stri
            if it == 4:
                proxyd["ssl"] = stri
            if it == 5:
                proxyd["location"] = stri
            if it == 6:
                proxyd["latency"] = stri
            if it == 7:
                proxyd["reliability"] = stri
            it += 1
            if it > 7:
                proxylist.append(proxyd)
                proxyd = {}
                it = 1
        return proxylist

xl = XroxyLib()
xl.print_lst(prefix='http://')
<|endoftext|>


<commit_before>"""
Extract client information from http user agent
The module does not try to detect all capabilities of browser in current form (it can easily be extended though).
Aim is
    * fast
    * very easy to extend
    * reliable enough for practical purposes
    * and assist python web apps to detect clients.
"""
import sys

class DetectorsHub(dict):
    _known_types = ['os', 'dist', 'flavor', 'browser']

    def __init__(self, *args, **kw):
        dict.__init__(self, *args, **kw)
        for typ in self._known_types:
            self.setdefault(typ, [])
        self.registerDetectors()

    def register(self, detector):
        if detector.info_type not in self._known_types:
            self[detector.info_type] = [detector]
            self._known_types.insert(detector.order, detector.info_type)
        else:
            self[detector.info_type].append(detector)

    def reorderByPrefs(self, detectors, prefs):
        if prefs is None:
            return []
        elif prefs == []:
            return detectors
        else:
            prefs.insert(0, '')
            return sorted(detectors,
                key=lambda d: d.name in prefs and prefs.index(d.name) or sys.maxsize)

    def __iter__(self):
        return iter(self._known_types)

    def registerDetectors(self):
        detectors = [v() for v in globals().values() if DetectorBase in getattr(v, '__mro__', [])]
        for d in detectors:
            if d.can_register:
                self.register(d)

class DetectorBase(object):
    name = "" # "to perform match in DetectorsHub object"
    info_type = "override me"
    result_key = "override me"
    order = 10 # 0 is highest
    look_for = "string to look for"
    skip_if_found = [] # strings if present stop processin
    can_register = False
    prefs = dict() # dict(info_type = [name1, name2], ..)
    version_splitters = ["/", " "]
    _suggested_detectors = None

    def __init__(self):
        if not self.name:
            self.name = self.__class__.__name__
        self.can_register = (self.__class__.__dict__.get('can_register', True))

    def detect(self, agent, result):
        # -> True/None
        if self.checkWords(agent):
            result[self.info_type] = dict(name=self.name)
            version = self.getVersion(agent)
            if version:
                result[self.info_type]['version'] = version
            return True

    def checkWords(self, agent):
        # -> True/None
        for w in self.skip_if_found:
            if w in agent:
                return False
        if self.look_for in agent:
            return True

    def getVersion(self, agent):
        # -> version string /None
        parts = agent.split(self.look_for + self.version_splitters[0])
        if len(parts) == 1:
            return None
        if len(self.version_splitters) == 1:
            return parts[1]
        return parts[min(len(parts) - 1, 1)].split(self.version_splitters[1])[0].strip()

class OS(DetectorBase):
    info_type = "os"
    can_register = False
    version_splitters = [";", " "]

class Dist(DetectorBase):
    info_type = "dist"
    can_register = False

class Flavor(DetectorBase):
    info_type = "flavor"
    can_register = False

class Browser(DetectorBase):
    info_type = "browser"
    can_register = False

class Macintosh(OS):
    look_for = 'Macintosh'
    prefs = dict(dist=None)

    def getVersion(self, agent): pass

class Firefox(Browser):
    look_for = "Firefox"
    skip_if_found = ["SeaMonkey"]

class SeaMonkey(Browser):
    look_for = "SeaMonkey"

class Konqueror(Browser):
    look_for = "Konqueror"
    version_splitters = ["/", ";"]

class OperaMobile(Browser):
    look_for = "Opera Mobi"
    name = "Opera Mobile"
    def getVersion(self, agent):
        try:
            look_for = "Version"
            return agent.split(look_for)[1][1:].split(' ')[0]
        except:
            look_for = "Opera"
            return agent.split(look_for)[1][1:].split(' ')[0]

class Opera(Browser):
    look_for = "Opera"
    def getVersion(self, agent):
        try:
            look_for = "Version"
            return agent.split(look_for)[1][1:].split(' ')[0]
        except:
            look_for = "Opera"
            return agent.split(look_for)[1][1:].split(' ')[0]

class Netscape(Browser):
    look_for = "Netscape"

class MSIE(Browser):
    look_for = "MSIE"
    skip_if_found = ["Opera"]
    name = "Microsoft Internet Explorer"
    version_splitters = [" ", ";"]

class Galeon(Browser):
    look_for = "Galeon"

class WOSBrowser(Browser):
    look_for = "wOSBrowser"

    def getVersion(self, agent): pass

class Safari(Browser):
    look_for = "Safari"

    def checkWords(self, agent):
        unless_list = ["Chrome", "OmniWeb", "wOSBrowser"]
        if self.look_for in agent:
            for word in unless_list:
                if word in agent:
                    return False
            return True

    def getVersion(self, agent):
        if "Version/" in agent:
            return agent.split('Version/')[-1].split(' ')[0].strip()
        else:
            return agent.split('Safari ')[-1].split(' ')[0].strip() # Mobile Safari

class Linux(OS):
    look_for = 'Linux'
    prefs = dict(browser=["Firefox"], dist=["Ubuntu", "Android"], flavor=None)

    def getVersion(self, agent): pass

class Blackberry(OS):
    look_for = 'BlackBerry'
    prefs = dict(dist=["BlackberryPlaybook"], flavor=None)
    def getVersion(self, agent): pass

class BlackberryPlaybook(Dist):
    look_for = 'PlayBook'
    def getVersion(self, agent): pass

class Macintosh(OS):
    look_for = 'Macintosh'
    prefs = dict(dist=None, flavor=['MacOS'])

    def getVersion(self, agent): pass

class MacOS(Flavor):
    look_for = 'Mac OS'
    prefs = dict(browser=['Firefox', 'Opera', "Microsoft Internet Explorer"])

    def getVersion(self, agent):
        version_end_chars = [';', ')']
        part = agent.split('Mac OS')[-1].strip()
        for c in version_end_chars:
            if c in part:
                version = part.split(c)[0]
                return version.replace('_', '.')
        return ''

class Windows(OS):
    look_for = 'Windows'
    prefs = dict(browser=["Microsoft Internet Explorer", 'Firefox'], dict=None, flavor=None)
    win_versions = {
            "NT 6.1": "7",
            "NT 6.0": "Vista",
            "NT 5.2": "Server 2003 / XP x64",
            "NT 5.1": "XP",
            "NT 5.01": "2000 SP1",
            "NT 5.0": "2000",
            "98; Win 9x 4.90": "Me"
        }

    def getVersion(self, agent):
        v = agent.split('Windows')[-1].split(';')[0].strip()
        if ')' in v:
            v = v.split(')')[0]
        v = self.win_versions.get(v, v)
        return v

class Ubuntu(Dist):
    look_for = 'Ubuntu'
    version_splitters = ["/", " "]
    prefs = dict(browser=['Firefox'])

class Debian(Dist):
    look_for = 'Debian'
    version_splitters = ["/", " "]
    prefs = dict(browser=['Firefox'])

class Chrome(Browser):
    look_for = "Chrome"
    version_splitters = ["/", " "]

class ChromeiOS(Browser):
    look_for = "CriOS"
    version_splitters = ["/", " "]

class ChromeOS(OS):
    look_for = "CrOS"
    version_splitters = [" ", " "]
    prefs = dict(browser=['Chrome'])
    def getVersion(self, agent):
        return agent.split(self.look_for + self.version_splitters[0])[-1].split(self.version_splitters[1])[1].strip()[:-1]

class Android(Dist):
    look_for = 'Android'
    prefs = dict(browser=['Safari'])

    def getVersion(self, agent):
        return agent.split(self.look_for)[-1].split(';')[0].strip()

class WebOS(Dist):
    look_for = 'hpwOS'

    def getVersion(self, agent):
        return agent.split('hpwOS/')[-1].split(';')[0].strip()

class IPhone(Dist):
    look_for = 'iPhone'
    prefs = dict(browser=['Safari'])

    def getVersion(self, agent):
        version_end_chars = [';', ')']
        part = agent.split('Mac OS')[-1].strip()
        for c in version_end_chars:
            if c in part:
                version = part.split(c)[0]
                return version.replace('_', '.')

class IPad(Dist):
    look_for = 'iPad'
    prefs = dict(browser=['Safari'])

    def getVersion(self, agent):
        version_end_chars = [';', ')']
        part = agent.split('Mac OS')[-1].strip()
        for c in version_end_chars:
            if c in part:
                version = part.split(c)[0]
                return version.replace('_', '.')

detectorshub = DetectorsHub()

def detect(agent):
    result = dict()
    prefs = dict()
    _suggested_detectors = []
    for info_type in detectorshub:
        if not _suggested_detectors:
            detectors = detectorshub[info_type]
            _d_prefs = prefs.get(info_type, [])
            detectors = detectorshub.reorderByPrefs(detectors, _d_prefs)
            if "detector" in locals():
                detector._suggested_detectors = detectors
        else:
            detectors = _suggested_detectors
        for detector in detectors:
            print("detector name: %s" % detector.name)
            if detector.detect(agent, result):
                prefs = detector.prefs
                _suggested_detectors = detector._suggested_detectors
                break
    return result

class Result(dict):
    def __missing__(self, k):
        return ""

def detect(agent):
    result = Result()
    _suggested_detectors = []
    for info_type in detectorshub:
        detectors = _suggested_detectors or detectorshub[info_type]
        for detector in detectors:
            if detector.detect(agent, result):
                if detector.prefs and not detector._suggested_detectors:
                    _suggested_detectors = detectorshub.reorderByPrefs(detectors, detector.prefs.get(info_type))
                    detector._suggested_detectors = _suggested_detectors
                    break
    return result

def simple_detect(agent):
    """
    -> (os, browser) # tuple of strings
    """
    result = detect(agent)
    os_list = []
    if 'flavor' in result: os_list.append(result['flavor']['name'])
    if 'dist' in result: os_list.append(result['dist']['name'])
    if 'os' in result: os_list.append(result['os']['name'])

    os = os_list and " ".join(os_list) or "Unknown OS"
    os_version = os_list and (result.get('flavor') and result['flavor'].get('version')) or (
    result.get('dist') and result['dist'].get('version')) or (result.get('os') and result['os'].get('version')) or ""
    browser = 'browser' in result and result['browser'].get('name') or 'Unknown Browser'
    browser_version = 'browser' in result and result['browser'].get('version') or ""
    if browser_version:
        browser = " ".join((browser, browser_version))
    if os_version:
        os = " ".join((os, os_version))
    return os, browser
<commit_msg>Add windows 8 string for windows NT 6.2<commit_after>"""
Extract client information from http user agent
The module does not try to detect all capabilities of browser in current form (it can easily be extended though).
Aim is
    * fast
    * very easy to extend
    * reliable enough for practical purposes
    * and assist python web apps to detect clients.
"""
import sys

class DetectorsHub(dict):
    _known_types = ['os', 'dist', 'flavor', 'browser']

    def __init__(self, *args, **kw):
        dict.__init__(self, *args, **kw)
        for typ in self._known_types:
            self.setdefault(typ, [])
        self.registerDetectors()

    def register(self, detector):
        if detector.info_type not in self._known_types:
            self[detector.info_type] = [detector]
            self._known_types.insert(detector.order, detector.info_type)
        else:
            self[detector.info_type].append(detector)

    def reorderByPrefs(self, detectors, prefs):
        if prefs is None:
            return []
        elif prefs == []:
            return detectors
        else:
            prefs.insert(0, '')
            return sorted(detectors,
                key=lambda d: d.name in prefs and prefs.index(d.name) or sys.maxsize)

    def __iter__(self):
        return iter(self._known_types)

    def registerDetectors(self):
        detectors = [v() for v in globals().values() if DetectorBase in getattr(v, '__mro__', [])]
        for d in detectors:
            if d.can_register:
                self.register(d)

class DetectorBase(object):
    name = "" # "to perform match in DetectorsHub object"
    info_type = "override me"
    result_key = "override me"
    order = 10 # 0 is highest
    look_for = "string to look for"
    skip_if_found = [] # strings if present stop processin
    can_register = False
    prefs = dict() # dict(info_type = [name1, name2], ..)
    version_splitters = ["/", " "]
    _suggested_detectors = None

    def __init__(self):
        if not self.name:
            self.name = self.__class__.__name__
        self.can_register = (self.__class__.__dict__.get('can_register', True))

    def detect(self, agent, result):
        # -> True/None
        if self.checkWords(agent):
            result[self.info_type] = dict(name=self.name)
            version = self.getVersion(agent)
            if version:
                result[self.info_type]['version'] = version
            return True

    def checkWords(self, agent):
        # -> True/None
        for w in self.skip_if_found:
            if w in agent:
                return False
        if self.look_for in agent:
            return True

    def getVersion(self, agent):
        # -> version string /None
        parts = agent.split(self.look_for + self.version_splitters[0])
        if len(parts) == 1:
            return None
        if len(self.version_splitters) == 1:
            return parts[1]
        return parts[min(len(parts) - 1, 1)].split(self.version_splitters[1])[0].strip()

class OS(DetectorBase):
    info_type = "os"
    can_register = False
    version_splitters = [";", " "]

class Dist(DetectorBase):
    info_type = "dist"
    can_register = False

class Flavor(DetectorBase):
    info_type = "flavor"
    can_register = False

class Browser(DetectorBase):
    info_type = "browser"
    can_register = False

class Macintosh(OS):
    look_for = 'Macintosh'
    prefs = dict(dist=None)

    def getVersion(self, agent): pass

class Firefox(Browser):
    look_for = "Firefox"
    skip_if_found = ["SeaMonkey"]

class SeaMonkey(Browser):
    look_for = "SeaMonkey"

class Konqueror(Browser):
    look_for = "Konqueror"
    version_splitters = ["/", ";"]

class OperaMobile(Browser):
    look_for = "Opera Mobi"
    name = "Opera Mobile"
    def getVersion(self, agent):
        try:
            look_for = "Version"
            return agent.split(look_for)[1][1:].split(' ')[0]
        except:
            look_for = "Opera"
            return agent.split(look_for)[1][1:].split(' ')[0]

class Opera(Browser):
    look_for = "Opera"
    def getVersion(self, agent):
        try:
            look_for = "Version"
            return agent.split(look_for)[1][1:].split(' ')[0]
        except:
            look_for = "Opera"
            return agent.split(look_for)[1][1:].split(' ')[0]

class Netscape(Browser):
    look_for = "Netscape"

class MSIE(Browser):
    look_for = "MSIE"
    skip_if_found = ["Opera"]
    name = "Microsoft Internet Explorer"
    version_splitters = [" ", ";"]

class Galeon(Browser):
    look_for = "Galeon"

class WOSBrowser(Browser):
    look_for = "wOSBrowser"

    def getVersion(self, agent): pass

class Safari(Browser):
    look_for = "Safari"

    def checkWords(self, agent):
        unless_list = ["Chrome", "OmniWeb", "wOSBrowser"]
        if self.look_for in agent:
            for word in unless_list:
                if word in agent:
                    return False
            return True

    def getVersion(self, agent):
        if "Version/" in agent:
            return agent.split('Version/')[-1].split(' ')[0].strip()
        else:
            return agent.split('Safari ')[-1].split(' ')[0].strip() # Mobile Safari

class Linux(OS):
    look_for = 'Linux'
    prefs = dict(browser=["Firefox"], dist=["Ubuntu", "Android"], flavor=None)

    def getVersion(self, agent): pass

class Blackberry(OS):
    look_for = 'BlackBerry'
    prefs = dict(dist=["BlackberryPlaybook"], flavor=None)
    def getVersion(self, agent): pass

class BlackberryPlaybook(Dist):
    look_for = 'PlayBook'
    def getVersion(self, agent): pass

class Macintosh(OS):
    look_for = 'Macintosh'
    prefs = dict(dist=None, flavor=['MacOS'])

    def getVersion(self, agent): pass

class MacOS(Flavor):
    look_for = 'Mac OS'
    prefs = dict(browser=['Firefox', 'Opera', "Microsoft Internet Explorer"])

    def getVersion(self, agent):
        version_end_chars = [';', ')']
        part = agent.split('Mac OS')[-1].strip()
        for c in version_end_chars:
            if c in part:
                version = part.split(c)[0]
                return version.replace('_', '.')
        return ''

class Windows(OS):
    look_for = 'Windows'
    prefs = dict(browser=["Microsoft Internet Explorer", 'Firefox'], dict=None, flavor=None)
    win_versions = {
            "NT 6.2": "8",
            "NT 6.1": "7",
            "NT 6.0": "Vista",
            "NT 5.2": "Server 2003 / XP x64",
            "NT 5.1": "XP",
            "NT 5.01": "2000 SP1",
            "NT 5.0": "2000",
            "98; Win 9x 4.90": "Me"
        }

    def getVersion(self, agent):
        v = agent.split('Windows')[-1].split(';')[0].strip()
        if ')' in v:
            v = v.split(')')[0]
        v = self.win_versions.get(v, v)
        return v

class Ubuntu(Dist):
    look_for = 'Ubuntu'
    version_splitters = ["/", " "]
    prefs = dict(browser=['Firefox'])

class Debian(Dist):
    look_for = 'Debian'
    version_splitters = ["/", " "]
    prefs = dict(browser=['Firefox'])

class Chrome(Browser):
    look_for = "Chrome"
    version_splitters = ["/", " "]

class ChromeiOS(Browser):
    look_for = "CriOS"
    version_splitters = ["/", " "]

class ChromeOS(OS):
    look_for = "CrOS"
    version_splitters = [" ", " "]
    prefs = dict(browser=['Chrome'])
    def getVersion(self, agent):
        return agent.split(self.look_for + self.version_splitters[0])[-1].split(self.version_splitters[1])[1].strip()[:-1]

class Android(Dist):
    look_for = 'Android'
    prefs = dict(browser=['Safari'])

    def getVersion(self, agent):
        return agent.split(self.look_for)[-1].split(';')[0].strip()

class WebOS(Dist):
    look_for = 'hpwOS'

    def getVersion(self, agent):
        return agent.split('hpwOS/')[-1].split(';')[0].strip()

class IPhone(Dist):
    look_for = 'iPhone'
    prefs = dict(browser=['Safari'])

    def getVersion(self, agent):
        version_end_chars = [';', ')']
        part = agent.split('Mac OS')[-1].strip()
        for c in version_end_chars:
            if c in part:
                version = part.split(c)[0]
                return version.replace('_', '.')

class IPad(Dist):
    look_for = 'iPad'
    prefs = dict(browser=['Safari'])

    def getVersion(self, agent):
        version_end_chars = [';', ')']
        part = agent.split('Mac OS')[-1].strip()
        for c in version_end_chars:
            if c in part:
                version = part.split(c)[0]
                return version.replace('_', '.')

detectorshub = DetectorsHub()

def detect(agent):
    result = dict()
    prefs = dict()
    _suggested_detectors = []
    for info_type in detectorshub:
        if not _suggested_detectors:
            detectors = detectorshub[info_type]
            _d_prefs = prefs.get(info_type, [])
            detectors = detectorshub.reorderByPrefs(detectors, _d_prefs)
            if "detector" in locals():
                detector._suggested_detectors = detectors
        else:
            detectors = _suggested_detectors
        for detector in detectors:
            print("detector name: %s" % detector.name)
            if detector.detect(agent, result):
                prefs = detector.prefs
                _suggested_detectors = detector._suggested_detectors
                break
    return result

class Result(dict):
    def __missing__(self, k):
        return ""

def detect(agent):
    result = Result()
    _suggested_detectors = []
    for info_type in detectorshub:
        detectors = _suggested_detectors or detectorshub[info_type]
        for detector in detectors:
            if detector.detect(agent, result):
                if detector.prefs and not detector._suggested_detectors:
                    _suggested_detectors = detectorshub.reorderByPrefs(detectors, detector.prefs.get(info_type))
                    detector._suggested_detectors = _suggested_detectors
                    break
    return result

def simple_detect(agent):
    """
    -> (os, browser) # tuple of strings
    """
    result = detect(agent)
    os_list = []
    if 'flavor' in result: os_list.append(result['flavor']['name'])
    if 'dist' in result: os_list.append(result['dist']['name'])
    if 'os' in result: os_list.append(result['os']['name'])

    os = os_list and " ".join(os_list) or "Unknown OS"
    os_version = os_list and (result.get('flavor') and result['flavor'].get('version')) or (
    result.get('dist') and result['dist'].get('version')) or (result.get('os') and result['os'].get('version')) or ""
    browser = 'browser' in result and result['browser'].get('name') or 'Unknown Browser'
    browser_version = 'browser' in result and result['browser'].get('version') or ""
    if browser_version:
        browser = " ".join((browser, browser_version))
    if os_version:
        os = " ".join((os, os_version))
    return os, browser
<|endoftext|>


<commit_before>import time
import socket
import random
import logging
import warnings
from gzip import GzipFile
from itertools import count
import requests
from requests.compat import StringIO
from collections import deque
from Queue import Queue
from threading import Thread, Event
from .utils import xauth, iterqueue
from .serialization import jsonencode

logger = logging.getLogger('hubstorage.batchuploader')

class BatchUploader(object):

    # Wait time between all batches status checks
    worker_loop_delay = 1.0
    # Max number of retries attempts before giving up,
    # Using 10 minutes max interval it accounts for 30hs total wait time
    worker_max_retries = 200
    # Minimum and maximum wait time between retries in seconds,
    # it can be 50% greater due to randomization
    worker_min_interval = 30
    worker_max_interval = 600

    def __init__(self, client):
        self.client = client
        self.closed = False
        self._wait_event = Event()
        self._writers = deque()
        self._thread = Thread(target=self._worker)
        self._thread.daemon = True
        self._thread.start()

    def create_writer(self, url, start=0, auth=None, size=1000, interval=15,
                      qsize=None, content_encoding='identity',
                      maxitemsize=1024 ** 2, callback=None):
        assert not self.closed, 'Can not create new writers when closed'
        auth = xauth(auth) or self.client.auth
        w = _BatchWriter(url=url,
                         auth=auth,
                         size=size,
                         start=start,
                         interval=interval,
                         qsize=qsize,
                         maxitemsize=maxitemsize,
                         content_encoding=content_encoding,
                         uploader=self,
                         callback=callback)
        self._writers.append(w)
        return w

    def close(self, timeout=None):
        self.closed = True
        self.interrupt()
        self._thread.join(timeout)

    def interrupt(self):
        self._wait_event.set()

    def __del__(self):
        if not self.closed:
            warnings.warn("%r not closed properly, some items may have been "
                          "lost!: %r" % (self.__class__.__name__, self._writers))

    def _interruptable_sleep(self):
        self._wait_event.wait(self.worker_loop_delay)
        self._wait_event.clear()

    def _worker(self):
        ctr = count()
        while True:
            if not self._writers:
                # Stop thread if closed and idle, but if open wait for writers
                if self.closed:
                    break
                self._interruptable_sleep()
                continue

            # Delay once all writers are processed
            if (ctr.next() % len(self._writers) == 0) and not self.closed:
                self._interruptable_sleep()

            # Get next writer to process
            w = self._writers.popleft()

            # Close open writers if uploader is closed
            if self.closed and not w.closed:
                w.close(block=False)

            # Checkpoint writer if eligible
            now = time.time()
            if w.itemsq.qsize() >= w.size or w.closed or w.flushme \
                    or w.checkpoint < now - w.interval:
                self._checkpoint(w)
                w.checkpoint = now

            # Re-queue pending or open writers
            if not (w.closed and w.itemsq.empty()):
                self._writers.append(w)

    def _checkpoint(self, w):
        q = w.itemsq
        qiter = iterqueue(q, w.size)
        data = self._content_encode(qiter, w)
        if qiter.count > 0:
            response = self._tryupload({
                'url': w.url,
                'offset': w.offset,
                'data': data,
                'auth': w.auth,
                'content-encoding': w.content_encoding,
            })
            w.offset += qiter.count
            for _ in xrange(qiter.count):
                q.task_done()
            if w.callback is not None:
                try:
                    w.callback(response)
                except Exception, e:
                    logger.exception("Callback for %s failed", w.url)

    def _content_encode(self, qiter, w):
        ce = w.content_encoding
        if ce == 'identity':
            return _encode_identity(qiter)
        elif ce == 'gzip':
            return _encode_gzip(qiter)
        else:
            raise ValueError('Writer using unknown content encoding: %s' % ce)

    def _tryupload(self, batch):
        """Retry uploads in case of server failures

        Use polinomial backoff with 10 minutes maximum interval that accounts
        for ~30 hours of total retry time.

        >>> sum(min(x**2, 600) for x in xrange(200)) / 3600
        30
        """
        url = batch['url']
        offset = batch['offset']
        for retryn in xrange(self.worker_max_retries):
            emsg = ''
            try:
                r = self._upload(batch)
                r.raise_for_status()
                if not (200 <= r.status_code < 300):
                    logger.warning('Discarding write to url=%s offset=%s: '
                                   '[HTTP error %s] %s\n%s', url, offset,
                                   r.status_code, r.reason, r.text.rstrip())
                return r
            except (socket.error, requests.RequestException) as e:
                if isinstance(e, requests.HTTPError):
                    emsg = "[HTTP error {0}] {1}".format(e.response.status_code,
                                                         e.response.text.rstrip())
                else:
                    emsg = str(e)
                logger.info("Retrying url=%s offset=%s: %s", url, offset, emsg)
            except Exception:
                logger.exception('Non retryable failure on url=%s offset=%s',
                                 url, offset)
                break

            backoff = min(max(retryn ** 2, self.worker_min_interval),
                          self.worker_max_interval)
            time.sleep(backoff * (0.5 + random.random()))

    def _upload(self, batch):
        params = {'start': batch['offset']}
        headers = {'content-encoding': batch['content-encoding']}
        return self.client.session.request(
            method='POST',
            url=batch['url'],
            data=batch['data'],
            auth=batch['auth'],
            timeout=self.client.connection_timeout,
            params=params,
            headers=headers,
        )

class ValueTooLarge(ValueError):
    """Raised when a serialized item is greater than 1MB"""

class _BatchWriter(object):
    #: Truncate overly big items to that many bytes for the error message.
    ERRMSG_DATA_TRUNCATION_LEN = 1024

    def __init__(self, url, start, auth, size, interval, qsize,
                 maxitemsize, content_encoding, uploader, callback=None):
        self.url = url
        self.offset = start
        self._nextid = count(start)
        self.auth = auth
        self.size = size
        self.interval = interval
        self.maxitemsize = maxitemsize
        self.content_encoding = content_encoding
        self.checkpoint = time.time()
        self.itemsq = Queue(size * 2 if qsize is None else qsize)
        self.closed = False
        self.flushme = False
        self.uploader = uploader
        self.callback = callback

    def write(self, item):
        assert not self.closed, 'attempting writes to a closed writer'
        data = jsonencode(item)
        if len(data) > self.maxitemsize:
            truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..."
            raise ValueTooLarge(
                'Value exceeds max encoded size of {} bytes: {!r}'
                .format(self.maxitemsize, truncated_data))

        self.itemsq.put(data)
        if self.itemsq.full():
            self.uploader.interrupt()
        return self._nextid.next()

    def flush(self):
        self.flushme = True
        self._waitforq()
        self.flushme = False

    def close(self, block=True):
        self.closed = True
        if block:
            self._waitforq()

    def _waitforq(self):
        self.uploader.interrupt()
        self.itemsq.join()

    def __str__(self):
        return self.url

def _encode_identity(iter):
    data = StringIO()
    for item in iter:
        data.write(item)
        data.write('\n')
    return data.getvalue()

def _encode_gzip(iter):
    data = StringIO()
    with GzipFile(fileobj=data, mode='w') as gzo:
        for item in iter:
            gzo.write(item)
            gzo.write('\n')
    return data.getvalue()
<commit_msg>batchuploader: explain total delay value<commit_after>import time
import socket
import random
import logging
import warnings
from gzip import GzipFile
from itertools import count
import requests
from requests.compat import StringIO
from collections import deque
from Queue import Queue
from threading import Thread, Event
from .utils import xauth, iterqueue
from .serialization import jsonencode

logger = logging.getLogger('hubstorage.batchuploader')

class BatchUploader(object):

    # Wait time between all batches status checks
    worker_loop_delay = 1.0
    # Max number of retry attempts before giving up.
    worker_max_retries = 200
    # The delay increases exponentially with the number of attempts but is
    # bounded with these values on both sides.
    worker_min_interval = 30
    worker_max_interval = 600
    # Each delay is also randomized by multiplying by random(0.5, 1.5), so the
    # total delay using the current parameters is an almost gaussian random
    # number with the following characteristics (see Irwin-Hall distribution):
    #
    # - average = 30hrs
    # - minimum = 15hrs
    # - maximum = 45hrs
    # - standard deviation = approx. 40m, which means that 95% of the time the
    #   total delay will be within 2*std = 1h20m of the average.

    def __init__(self, client):
        self.client = client
        self.closed = False
        self._wait_event = Event()
        self._writers = deque()
        self._thread = Thread(target=self._worker)
        self._thread.daemon = True
        self._thread.start()

    def create_writer(self, url, start=0, auth=None, size=1000, interval=15,
                      qsize=None, content_encoding='identity',
                      maxitemsize=1024 ** 2, callback=None):
        assert not self.closed, 'Can not create new writers when closed'
        auth = xauth(auth) or self.client.auth
        w = _BatchWriter(url=url,
                         auth=auth,
                         size=size,
                         start=start,
                         interval=interval,
                         qsize=qsize,
                         maxitemsize=maxitemsize,
                         content_encoding=content_encoding,
                         uploader=self,
                         callback=callback)
        self._writers.append(w)
        return w

    def close(self, timeout=None):
        self.closed = True
        self.interrupt()
        self._thread.join(timeout)

    def interrupt(self):
        self._wait_event.set()

    def __del__(self):
        if not self.closed:
            warnings.warn("%r not closed properly, some items may have been "
                          "lost!: %r" % (self.__class__.__name__, self._writers))

    def _interruptable_sleep(self):
        self._wait_event.wait(self.worker_loop_delay)
        self._wait_event.clear()

    def _worker(self):
        ctr = count()
        while True:
            if not self._writers:
                # Stop thread if closed and idle, but if open wait for writers
                if self.closed:
                    break
                self._interruptable_sleep()
                continue

            # Delay once all writers are processed
            if (ctr.next() % len(self._writers) == 0) and not self.closed:
                self._interruptable_sleep()

            # Get next writer to process
            w = self._writers.popleft()

            # Close open writers if uploader is closed
            if self.closed and not w.closed:
                w.close(block=False)

            # Checkpoint writer if eligible
            now = time.time()
            if w.itemsq.qsize() >= w.size or w.closed or w.flushme \
                    or w.checkpoint < now - w.interval:
                self._checkpoint(w)
                w.checkpoint = now

            # Re-queue pending or open writers
            if not (w.closed and w.itemsq.empty()):
                self._writers.append(w)

    def _checkpoint(self, w):
        q = w.itemsq
        qiter = iterqueue(q, w.size)
        data = self._content_encode(qiter, w)
        if qiter.count > 0:
            response = self._tryupload({
                'url': w.url,
                'offset': w.offset,
                'data': data,
                'auth': w.auth,
                'content-encoding': w.content_encoding,
            })
            w.offset += qiter.count
            for _ in xrange(qiter.count):
                q.task_done()
            if w.callback is not None:
                try:
                    w.callback(response)
                except Exception, e:
                    logger.exception("Callback for %s failed", w.url)

    def _content_encode(self, qiter, w):
        ce = w.content_encoding
        if ce == 'identity':
            return _encode_identity(qiter)
        elif ce == 'gzip':
            return _encode_gzip(qiter)
        else:
            raise ValueError('Writer using unknown content encoding: %s' % ce)

    def _tryupload(self, batch):
        """Retry uploads in case of server failures

        Use polinomial backoff with 10 minutes maximum interval that accounts
        for ~30 hours of total retry time.

        >>> sum(min(x**2, 600) for x in xrange(200)) / 3600
        30
        """
        url = batch['url']
        offset = batch['offset']
        for retryn in xrange(self.worker_max_retries):
            emsg = ''
            try:
                r = self._upload(batch)
                r.raise_for_status()
                if not (200 <= r.status_code < 300):
                    logger.warning('Discarding write to url=%s offset=%s: '
                                   '[HTTP error %s] %s\n%s', url, offset,
                                   r.status_code, r.reason, r.text.rstrip())
                return r
            except (socket.error, requests.RequestException) as e:
                if isinstance(e, requests.HTTPError):
                    emsg = "[HTTP error {0}] {1}".format(e.response.status_code,
                                                         e.response.text.rstrip())
                else:
                    emsg = str(e)
                logger.info("Retrying url=%s offset=%s: %s", url, offset, emsg)
            except Exception:
                logger.exception('Non retryable failure on url=%s offset=%s',
                                 url, offset)
                break

            backoff = min(max(retryn ** 2, self.worker_min_interval),
                          self.worker_max_interval)
            time.sleep(backoff * (0.5 + random.random()))

    def _upload(self, batch):
        params = {'start': batch['offset']}
        headers = {'content-encoding': batch['content-encoding']}
        return self.client.session.request(
            method='POST',
            url=batch['url'],
            data=batch['data'],
            auth=batch['auth'],
            timeout=self.client.connection_timeout,
            params=params,
            headers=headers,
        )

class ValueTooLarge(ValueError):
    """Raised when a serialized item is greater than 1MB"""

class _BatchWriter(object):
    #: Truncate overly big items to that many bytes for the error message.
    ERRMSG_DATA_TRUNCATION_LEN = 1024

    def __init__(self, url, start, auth, size, interval, qsize,
                 maxitemsize, content_encoding, uploader, callback=None):
        self.url = url
        self.offset = start
        self._nextid = count(start)
        self.auth = auth
        self.size = size
        self.interval = interval
        self.maxitemsize = maxitemsize
        self.content_encoding = content_encoding
        self.checkpoint = time.time()
        self.itemsq = Queue(size * 2 if qsize is None else qsize)
        self.closed = False
        self.flushme = False
        self.uploader = uploader
        self.callback = callback

    def write(self, item):
        assert not self.closed, 'attempting writes to a closed writer'
        data = jsonencode(item)
        if len(data) > self.maxitemsize:
            truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..."
            raise ValueTooLarge(
                'Value exceeds max encoded size of {} bytes: {!r}'
                .format(self.maxitemsize, truncated_data))

        self.itemsq.put(data)
        if self.itemsq.full():
            self.uploader.interrupt()
        return self._nextid.next()

    def flush(self):
        self.flushme = True
        self._waitforq()
        self.flushme = False

    def close(self, block=True):
        self.closed = True
        if block:
            self._waitforq()

    def _waitforq(self):
        self.uploader.interrupt()
        self.itemsq.join()

    def __str__(self):
        return self.url

def _encode_identity(iter):
    data = StringIO()
    for item in iter:
        data.write(item)
        data.write('\n')
    return data.getvalue()

def _encode_gzip(iter):
    data = StringIO()
    with GzipFile(fileobj=data, mode='w') as gzo:
        for item in iter:
            gzo.write(item)
            gzo.write('\n')
    return data.getvalue()
<|endoftext|>


<commit_before><commit_msg>rm unused<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from hwt.code import If, Or, log2ceil
from hwt.hdl.types.bits import Bits
from hwt.interfaces.std import Signal, VldSynced, VectSignal
from hwt.serializer.mode import serializeParamsUniq
from hwt.synthesizer.unit import Unit
from hwt.synthesizer.param import Param
from hwt.synthesizer.rtlLevel.rtlSignal import RtlSignal
from hwt.synthesizer.vectorUtils import iterBits

@serializeParamsUniq
class OneHotToBin(Unit):
    """
    Converts one hot signal to binary, bin.vld is high when oneHot != 0
    """

    def _config(self):
        self.ONE_HOT_WIDTH = Param(8)

    def _declr(self):
        self.oneHot = VectSignal(self.ONE_HOT_WIDTH)
        self.bin = VldSynced()
        self.bin.DATA_WIDTH.set(log2ceil(self.ONE_HOT_WIDTH))

    def _impl(self):
        self.bin.data(oneHotToBin(self, self.oneHot))
        self.bin.vld(Or(*[bit for bit in iterBits(self.oneHot)]))

def oneHotToBin(parent, signals, resName="oneHotToBin"):
    if isinstance(signals, (RtlSignal, Signal)):
        signals = [signals[i] for i in range(signals._dtype.bit_length())]
    else:
        signals = list(signals)

    res = parent._sig(resName, Bits(log2ceil(len(signals))))
    leadingZeroTop = None
    for i, s in enumerate(reversed(signals)):
        connections = res(len(signals) - i - 1)

        if leadingZeroTop is None:
            leadingZeroTop = connections
        else:
            leadingZeroTop = If(s,
                                connections
                             ).Else(
                                leadingZeroTop
                             )

    return res

if __name__ == "__main__":
    from hwt.synthesizer.utils import toRtl
    u = OneHotToBin()
    print(toRtl(u))
<commit_msg>schematic in doc for OneHotToBin<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from hwt.code import If, Or, log2ceil
from hwt.hdl.types.bits import Bits
from hwt.interfaces.std import Signal, VldSynced, VectSignal
from hwt.serializer.mode import serializeParamsUniq
from hwt.synthesizer.unit import Unit
from hwt.synthesizer.param import Param
from hwt.synthesizer.rtlLevel.rtlSignal import RtlSignal
from hwt.synthesizer.vectorUtils import iterBits

@serializeParamsUniq
class OneHotToBin(Unit):
    """
    Converts one hot signal to binary, bin.vld is high when oneHot != 0

    .. hwt-schematic::
    """

    def _config(self):
        self.ONE_HOT_WIDTH = Param(8)

    def _declr(self):
        self.oneHot = VectSignal(self.ONE_HOT_WIDTH)
        self.bin = VldSynced()
        self.bin.DATA_WIDTH.set(log2ceil(self.ONE_HOT_WIDTH))

    def _impl(self):
        self.bin.data(oneHotToBin(self, self.oneHot))
        self.bin.vld(Or(*[bit for bit in iterBits(self.oneHot)]))

def oneHotToBin(parent, signals, resName="oneHotToBin"):
    if isinstance(signals, (RtlSignal, Signal)):
        signals = [signals[i] for i in range(signals._dtype.bit_length())]
    else:
        signals = list(signals)

    res = parent._sig(resName, Bits(log2ceil(len(signals))))
    leadingZeroTop = None
    for i, s in enumerate(reversed(signals)):
        connections = res(len(signals) - i - 1)

        if leadingZeroTop is None:
            leadingZeroTop = connections
        else:
            leadingZeroTop = If(s,
                                connections
                             ).Else(
                                leadingZeroTop
                             )

    return res

if __name__ == "__main__":
    from hwt.synthesizer.utils import toRtl
    u = OneHotToBin()
    print(toRtl(u))
<|endoftext|>


<commit_before>"""
awslimitchecker/services/ecs.py

The latest version of this package is available at:
<https://github.com/di1214/awslimitchecker>

################################################################################
Copyright 2015-2017 Di Zou <zou@pythian.com>

    This file is part of awslimitchecker, also known as awslimitchecker.

    awslimitchecker is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    awslimitchecker is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with awslimitchecker.  If not, see <http://www.gnu.org/licenses/>.

The Copyright and Authors attributions contained herein may not be removed or
otherwise altered, except to add the Author attribution of a contributor to
this work. (Additional Terms pursuant to Section 7b of the AGPL v3)
################################################################################
While not legally required, I sincerely request that anyone who finds
bugs please submit them at <https://github.com/di1214/pydnstest> or
to me via email, and that you send any contributions or improvements
either as a pull request on GitHub, or to me via email.
################################################################################

AUTHORS:
Di Zou <zou@pythian.com>
################################################################################
"""

import sys
from awslimitchecker.services.directoryservice import _DirectoryserviceService

# https://code.google.com/p/mock/issues/detail?id=249
# py>=3.4 should use unittest.mock not the mock package on pypi
if (
        sys.version_info[0] < 3 or
        sys.version_info[0] == 3 and sys.version_info[1] < 4
):
    from mock import patch, call, Mock
else:
    from unittest.mock import patch, call, Mock

pbm = 'awslimitchecker.services.directoryservice'  # module patch base
pb = '%s._DirectoryserviceService' % pbm  # class patch pase

class Test_DirectoryserviceService(object):

    def test_init(self):
        """test __init__()"""
        cls = _DirectoryserviceService(21, 43)
        assert cls.service_name == 'Directory Service'
        assert cls.api_name == 'ds'
        assert cls.conn is None
        assert cls.warning_threshold == 21
        assert cls.critical_threshold == 43

    def test_get_limits(self):
        cls = _DirectoryserviceService(21, 43)
        cls.limits = {}
        res = cls.get_limits()
        assert sorted(res.keys()) == sorted([
            'CloudOnlyDirectories',
            'CloudOnlyMicrosoftAD',
            'ConnectedDirectories',
        ])
        for name, limit in res.items():
            assert limit.service == cls
            assert limit.def_warning_threshold == 21
            assert limit.def_critical_threshold == 43

    def test_get_limits_again(self):
        """test that existing limits dict is returned on subsequent calls"""
        mock_limits = Mock()
        cls = _DirectoryserviceService(21, 43)
        cls.limits = mock_limits
        res = cls.get_limits()
        assert res == mock_limits

    def test_find_usage(self):
        mock_conn = Mock()
        mock_conn.get_directory_limits.return_value = {
            'DirectoryLimits': {
                'CloudOnlyDirectoriesLimit': 123,
                'CloudOnlyDirectoriesCurrentCount': 123,
                'CloudOnlyDirectoriesLimitReached': True | False,
                'CloudOnlyMicrosoftADLimit': 123,
                'CloudOnlyMicrosoftADCurrentCount': 123,
                'CloudOnlyMicrosoftADLimitReached': True | False,
                'ConnectedDirectoriesLimit': 123,
                'ConnectedDirectoriesCurrentCount': 123,
                'ConnectedDirectoriesLimitReached': True | False
            }
        }
        with patch('%s.connect' % pb) as mock_connect:
            cls = _DirectoryserviceService(21, 43)
            cls.conn = mock_conn
            assert cls._have_usage is False
            cls.find_usage()
        assert mock_connect.mock_calls == [call()]
        assert cls._have_usage is True
        assert mock_conn.mock_calls == [call.get_directory_limits()]
        assert cls.limits['CloudOnlyDirectories'].get_current_usage()[
            0].get_value() == 123

    def test_required_iam_permissions(self):
        cls = _DirectoryserviceService(21, 43)
        assert cls.required_iam_permissions() == [
            'ds:GetDirectoryLimits'
        ]
<commit_msg>more pep8 clean up<commit_after>"""
awslimitchecker/services/ecs.py

The latest version of this package is available at:
<https://github.com/di1214/awslimitchecker>

################################################################################
Copyright 2015-2017 Di Zou <zou@pythian.com>

    This file is part of awslimitchecker, also known as awslimitchecker.

    awslimitchecker is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    awslimitchecker is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with awslimitchecker.  If not, see <http://www.gnu.org/licenses/>.

The Copyright and Authors attributions contained herein may not be removed or
otherwise altered, except to add the Author attribution of a contributor to
this work. (Additional Terms pursuant to Section 7b of the AGPL v3)
################################################################################
While not legally required, I sincerely request that anyone who finds
bugs please submit them at <https://github.com/di1214/pydnstest> or
to me via email, and that you send any contributions or improvements
either as a pull request on GitHub, or to me via email.
################################################################################

AUTHORS:
Di Zou <zou@pythian.com>
################################################################################
"""

import sys
from awslimitchecker.services.directoryservice import _DirectoryserviceService

# https://code.google.com/p/mock/issues/detail?id=249
# py>=3.4 should use unittest.mock not the mock package on pypi
if (
        sys.version_info[0] < 3 or
        sys.version_info[0] == 3 and sys.version_info[1] < 4
):
    from mock import patch, call, Mock
else:
    from unittest.mock import patch, call, Mock

pbm = 'awslimitchecker.services.directoryservice'  # module patch base
pb = '%s._DirectoryserviceService' % pbm  # class patch pase

class Test_DirectoryserviceService(object):

    def test_init(self):
        """test __init__()"""
        cls = _DirectoryserviceService(21, 43)
        assert cls.service_name == 'Directory Service'
        assert cls.api_name == 'ds'
        assert cls.conn is None
        assert cls.warning_threshold == 21
        assert cls.critical_threshold == 43

    def test_get_limits(self):
        cls = _DirectoryserviceService(21, 43)
        cls.limits = {}
        res = cls.get_limits()
        assert sorted(res.keys()) == sorted([
            'CloudOnlyDirectories',
            'CloudOnlyMicrosoftAD',
            'ConnectedDirectories',
        ])
        for name, limit in res.items():
            assert limit.service == cls
            assert limit.def_warning_threshold == 21
            assert limit.def_critical_threshold == 43

    def test_get_limits_again(self):
        """test that existing limits dict is returned on subsequent calls"""
        mock_limits = Mock()
        cls = _DirectoryserviceService(21, 43)
        cls.limits = mock_limits
        res = cls.get_limits()
        assert res == mock_limits

    def test_find_usage(self):
        mock_conn = Mock()
        mock_conn.get_directory_limits.return_value = {
            'DirectoryLimits': {
                'CloudOnlyDirectoriesLimit': 123,
                'CloudOnlyDirectoriesCurrentCount': 123,
                'CloudOnlyDirectoriesLimitReached': True | False,
                'CloudOnlyMicrosoftADLimit': 123,
                'CloudOnlyMicrosoftADCurrentCount': 123,
                'CloudOnlyMicrosoftADLimitReached': True | False,
                'ConnectedDirectoriesLimit': 123,
                'ConnectedDirectoriesCurrentCount': 123,
                'ConnectedDirectoriesLimitReached': True | False
            }
        }
        with patch('%s.connect' % pb) as mock_connect:
            cls = _DirectoryserviceService(21, 43)
            cls.conn = mock_conn
            assert cls._have_usage is False
            cls.find_usage()
        assert mock_connect.mock_calls == [call()]
        assert cls._have_usage is True
        assert mock_conn.mock_calls == [call.get_directory_limits()]
        assert cls.limits[
            'CloudOnlyDirectories'].get_current_usage()[
            0].get_value() == 123

    def test_required_iam_permissions(self):
        cls = _DirectoryserviceService(21, 43)
        assert cls.required_iam_permissions() == [
            'ds:GetDirectoryLimits'
        ]
<|endoftext|>