<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals
from __future__ import absolute_import

from noterator.config import load_config, ConfigParser, ConfigurationError
from noterator.plugins import email, hipchat, twilio
from noterator.utils import catch_all, now

__author__ = 'James Rutherford'
__email__ = 'jim@jimr.org'
__version__ = '0.2.2'

QUIET = 0
EMAIL = 1
TWILIO = 1 << 1
HIPCHAT = 1 << 2

class Noterator(object):
    """Base class for setting up & configuring your noteration.

    You only really need to interact with this class directly (rather than just
    using ``noterator.noterate``) is if you want to build a re-usable
    ``Noterator`` or you want to build one without using a configuration file.

    For example, you could define a ``Noterator`` for use on two iterables, and
    use a different description for each one::

        >>> from noterator import Noterator, EMAIL
        >>> noterator = Noterator(method=EMAIL, every_n=100, start=True)
        >>> for obj in noterator(my_objects, desc="loop 1")
        ...     do_something_slow(obj)
        ...
        >>> for obj in noterator(my_other_objects, desc="loop 2")
        ...     do_something_else_slow(obj)
        ...
        >>>

    And if you want to build a Noterator without using a configuration file (or
    if you just want to override some configured options)::

        >>> from noterator import Noterator, EMAIL
        >>> noterator = Noterator(my_objects, method=EMAIL, every_n=100)
        >>> noterator.configure_plugin('email', recipient='you@example.org')
        >>> for obj in noterator
        ...     do_something_slow(obj)
        ...
        >>>

    If `iterable` isn't set on class initialisation, it must be set before
    looping, or as a parameter to the ``noterate`` method.

    When iteration begins, the configuration will be validated.

    Args:
        Technically none, but ``iterable``, ``method``, and ``desc`` are often
        passed as positional arguments, and such usage is encouraged for
        brevity.

    Kwargs:
        iterable (iterable): The iterable we're going to wrap
        desc (str): Description to include in the notification.
        start (bool): Send a notification when iteration starts
        finish (bool): Send a notification when iteration completes
        every_n (int): Send a notification every ``every_n`` iterations
        method (int): Method(s) to use (e.g. ``EMAIL|HIPCHAT``)
        head (string): Header for notification message
        body (string): Body for notification message. We call `.format` on this
                       string with the iteration count at the point of
                       notification.
        config_file (string): Path to alternative configuration file.

    """
    index = 0
    methods = (
        (EMAIL, 'email', email),
        (TWILIO, 'twilio', twilio),
        (HIPCHAT, 'hipchat', hipchat),
    )

    def __init__(self, iterable=None, method=QUIET, desc=None, head=None,
                 body=None, start=False, finish=True, every_n=None,
                 config_file=None):
        self.iterable = iterable
        self.method = method

        self.desc = desc or 'Iteration'
        self.head = head or 'Noterator alert'
        self.body = body or 'progress: {} iterations.'

        self.start = start
        self.finish = finish
        self.every_n = every_n

        # If the configuration file isn't present, build an empty configuration
        # that can be populated with `self.configure_plugin`.
        self.cfg = ConfigParser()
        try:
            self.cfg = load_config(config_file)
        except IOError:
            pass

    def __call__(self, iterable=None, method=None, desc=None, **kwargs):
        """Allow any parameter to be set at execution time.

        For example::

            >>> from noterator import Noterator, EMAIL
            >>> noterator = Noterator(my_objects, EMAIL, every_n=100)
            >>> for obj in noterator(desc="loop 1")
            ...     do_something_slow(obj)
            ...

        """
        if iterable is not None:
            self.iterable = iterable

        if method is not None:
            self.method = method

        if desc is not None:
            self.desc = desc

        for key, value in kwargs.items():
            setattr(self, key, value)

        return self

    def __iter__(self):
        self._validate_config()
        self._notify(started=True)

        for obj in self.iterable:
            yield obj
            self._notify()
            self.index += 1

        self._notify(finished=True)

    def next(self):
        if self.index == 0:
            self._validate_config()
            self._notify(started=True)

        try:
            result = next(self.iterable)
            self.index += 1
            self._notify()
            return result
        except StopIteration:
            self._notify(finished=True)
            raise

    def configure_plugin(self, name, **kwargs):
        """Optionally set up plugins without using a configuration file.

        You can also use this to override defaults set in a configuration file.

        Args:
            name (str): the name of the plugin (e.g. 'email')

        Kwargs: any keyword arguments accepted by the named plugin.

        """
        if not self.cfg.has_section(name):
            self.cfg.add_section(name)

        for key, value in kwargs.items():
            self.cfg.set(name, key, value)

    def _validate_config(self):
        for flag, config_key, module in self.methods:
            if self.method & flag:
                # If we're using a given notification method, make sure it's
                # configured.
                if config_key not in self.cfg.sections():
                    raise ConfigurationError(
                        '{} is not configured'.format(config_key),
                    )
                else:
                    # Make sure all required configuration parameters are set
                    configured = set(self.cfg.options(config_key))
                    required = set(module.REQUIRED_CONFIG)
                    if configured & required != required:
                        raise ConfigurationError(
                            '{} is missing required settings: {}'.format(
                                config_key, ', '.join(required - configured),
                            )
                        )

    @catch_all
    def _notify(self, started=False, finished=False):
        send = False
        body = self.body.format(self.index)

        if self.start and started:
            send = True
            body = '{} started at {}.'.format(self.desc, now())

        if self.finish and finished:
            send = True
            body = '{} finished at {} (total iterations: {}).'.format(
                self.desc, now(), self.index,
            )
        elif self.every_n and self.index > 0 and self.index % self.every_n == 0:
            send = True
            body = '{} completed {} iterations at {}.'.format(
                self.desc, self.index, now(),
            )

        if send:
            for method, config_key, module in self.methods:
                if self.method & method:
                    module.notify(
                        self.head, body, **dict(self.cfg.items(config_key))
                    )

def noterate(iterable, *args, **kwargs):
    """Wrap any iterable with `noterate` and you'll be notified when the iteration
    completes, for example::

        >>> from noterator import noterate, EMAIL
        >>> for obj in noterate(my_objects, EMAIL, "My super-slow iteration"):
        ...     do_something_slow(obj)
        ...
        >>>

    By default you only find out when the iteration completes, but sometimes
    it's useful to know when these things start too::

        >>> from noterator import noterate, EMAIL, HIPCHAT
        >>> for obj in noterate(my_objects, EMAIL|HIPCHAT, start=True):
        ...     do_something_slow(obj)
        ...
        >>>

    This function is a convenience wrapper around the ``Noterator`` class. The
    code above is equivalent to::

        >>> from noterator import Noterator, EMAIL, HIPCHAT
        >>> noterator = Noterator(my_objects, EMAIL|HIPCHAT, start=True)
        >>> for obj in noterator:
        ...     do_something_slow(obj)
        ...
        >>>

    Positional and Keyword arguments are the same as for ``Noterator.__init__``
    except that ``iterable`` is required.

    """
    return Noterator(iterable, *args, **kwargs)
<commit_msg>Minor refactor for clarity<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals
from __future__ import absolute_import

from noterator.config import load_config, ConfigParser, ConfigurationError
from noterator.plugins import email, hipchat, twilio
from noterator.utils import catch_all, now

__author__ = 'James Rutherford'
__email__ = 'jim@jimr.org'
__version__ = '0.2.2'

QUIET = 0
EMAIL = 1
TWILIO = 1 << 1
HIPCHAT = 1 << 2

class Noterator(object):
    """Base class for setting up & configuring your noteration.

    You only really need to interact with this class directly (rather than just
    using ``noterator.noterate``) is if you want to build a re-usable
    ``Noterator`` or you want to build one without using a configuration file.

    For example, you could define a ``Noterator`` for use on two iterables, and
    use a different description for each one::

        >>> from noterator import Noterator, EMAIL
        >>> noterator = Noterator(method=EMAIL, every_n=100, start=True)
        >>> for obj in noterator(my_objects, desc="loop 1")
        ...     do_something_slow(obj)
        ...
        >>> for obj in noterator(my_other_objects, desc="loop 2")
        ...     do_something_else_slow(obj)
        ...
        >>>

    And if you want to build a Noterator without using a configuration file (or
    if you just want to override some configured options)::

        >>> from noterator import Noterator, EMAIL
        >>> noterator = Noterator(my_objects, method=EMAIL, every_n=100)
        >>> noterator.configure_plugin('email', recipient='you@example.org')
        >>> for obj in noterator
        ...     do_something_slow(obj)
        ...
        >>>

    If `iterable` isn't set on class initialisation, it must be set before
    looping, or as a parameter to the ``noterate`` method.

    When iteration begins, the configuration will be validated.

    Args:
        Technically none, but ``iterable``, ``method``, and ``desc`` are often
        passed as positional arguments, and such usage is encouraged for
        brevity.

    Kwargs:
        iterable (iterable): The iterable we're going to wrap
        desc (str): Description to include in the notification.
        start (bool): Send a notification when iteration starts
        finish (bool): Send a notification when iteration completes
        every_n (int): Send a notification every ``every_n`` iterations
        method (int): Method(s) to use (e.g. ``EMAIL|HIPCHAT``)
        head (string): Header for notification message
        body (string): Body for notification message. We call `.format` on this
                       string with the iteration count at the point of
                       notification.
        config_file (string): Path to alternative configuration file.

    """
    index = 0
    methods = (
        (EMAIL, 'email', email),
        (TWILIO, 'twilio', twilio),
        (HIPCHAT, 'hipchat', hipchat),
    )

    def __init__(self, iterable=None, method=QUIET, desc=None, head=None,
                 body=None, start=False, finish=True, every_n=None,
                 config_file=None):
        self.iterable = iterable
        self.method = method

        self.desc = desc or 'Iteration'
        self.head = head or 'Noterator alert'
        self.body = body or 'progress: {} iterations.'

        self.start = start
        self.finish = finish
        self.every_n = every_n

        # If the configuration file isn't present, build an empty configuration
        # that can be populated with `self.configure_plugin`.
        self.cfg = ConfigParser()
        try:
            self.cfg = load_config(config_file)
        except IOError:
            pass

    def __call__(self, iterable=None, method=None, desc=None, **kwargs):
        """Allow any parameter to be set at execution time.

        For example::

            >>> from noterator import Noterator, EMAIL
            >>> noterator = Noterator(my_objects, EMAIL, every_n=100)
            >>> for obj in noterator(desc="loop 1")
            ...     do_something_slow(obj)
            ...

        """
        if iterable is not None:
            self.iterable = iterable

        if method is not None:
            self.method = method

        if desc is not None:
            self.desc = desc

        for key, value in kwargs.items():
            setattr(self, key, value)

        return self

    def __iter__(self):
        self._validate_config()
        self._notify(started=True)

        for obj in self.iterable:
            yield obj
            self._notify()
            self.index += 1

        self._notify(finished=True)

    def next(self):
        if self.index == 0:
            self._validate_config()
            self._notify(started=True)

        try:
            result = next(self.iterable)
            self.index += 1
            self._notify()
            return result
        except StopIteration:
            self._notify(finished=True)
            raise

    def configure_plugin(self, name, **kwargs):
        """Optionally set up plugins without using a configuration file.

        You can also use this to override defaults set in a configuration file.

        Args:
            name (str): the name of the plugin (e.g. 'email')

        Kwargs: any keyword arguments accepted by the named plugin.

        """
        if not self.cfg.has_section(name):
            self.cfg.add_section(name)

        for key, value in kwargs.items():
            self.cfg.set(name, key, value)

    def _validate_config(self):
        for flag, config_key, module in self.methods:
            if self.method & flag:
                # If we're using a given notification method, make sure it's
                # configured.
                if config_key not in self.cfg.sections():
                    raise ConfigurationError(
                        '{} is not configured'.format(config_key),
                    )
                else:
                    # Make sure all required configuration parameters are set
                    configured = set(self.cfg.options(config_key))
                    required = set(module.REQUIRED_CONFIG)
                    if configured & required != required:
                        raise ConfigurationError(
                            '{} is missing required settings: {}'.format(
                                config_key, ', '.join(required - configured),
                            )
                        )

    @catch_all
    def _notify(self, started=False, finished=False):
        send = False
        body = self.body.format(self.index)

        if self.start and started:
            send = True
            body = '{} started at {}.'.format(self.desc, now())

        if self.finish and finished:
            send = True
            body = '{} finished at {} (total iterations: {}).'.format(
                self.desc, now(), self.index,
            )
        elif self.every_n:
            # We don't send `every_n` notifications on iteration completion
            # where that would be a duplicate notification hence the `elif`
            # with `self.finish and finished`
            if self.index > 0 and self.index % self.every_n == 0:
                send = True
                body = '{} completed {} iterations at {}.'.format(
                    self.desc, self.index, now(),
                )

        if send:
            for method, config_key, module in self.methods:
                if self.method & method:
                    module.notify(
                        self.head, body, **dict(self.cfg.items(config_key))
                    )

def noterate(iterable, *args, **kwargs):
    """Wrap any iterable with `noterate` and you'll be notified when the iteration
    completes, for example::

        >>> from noterator import noterate, EMAIL
        >>> for obj in noterate(my_objects, EMAIL, "My super-slow iteration"):
        ...     do_something_slow(obj)
        ...
        >>>

    By default you only find out when the iteration completes, but sometimes
    it's useful to know when these things start too::

        >>> from noterator import noterate, EMAIL, HIPCHAT
        >>> for obj in noterate(my_objects, EMAIL|HIPCHAT, start=True):
        ...     do_something_slow(obj)
        ...
        >>>

    This function is a convenience wrapper around the ``Noterator`` class. The
    code above is equivalent to::

        >>> from noterator import Noterator, EMAIL, HIPCHAT
        >>> noterator = Noterator(my_objects, EMAIL|HIPCHAT, start=True)
        >>> for obj in noterator:
        ...     do_something_slow(obj)
        ...
        >>>

    Positional and Keyword arguments are the same as for ``Noterator.__init__``
    except that ``iterable`` is required.

    """
    return Noterator(iterable, *args, **kwargs)
<|endoftext|>


<commit_before>import hashlib
import hmac
import binascii

'''
Python implementation of HOTP and TOTP algorithms from the OATH project.
'''

def __truncated_value(h):
    bytes = map(ord, h)
    offset = bytes[-1] & 0xf
    v = (bytes[offset] & 0x7f) << 24 | (bytes[offset+1] & 0xff) << 16 | \
            (bytes[offset+2] & 0xff) << 8 | (bytes[offset+3] & 0xff)
    return v

def dec(h,p):
    v = str(__truncated_value(h))
    return v[len(v)-p:]

def __hotp(key, counter, hash=hashlib.sha1):
    hex_counter = hex(long(counter))[2:-1]
    hex_counter = '0' * (16 - len(hex_counter)) + hex_counter
    bin_counter = binascii.unhexlify(hex_counter)
    bin_key = binascii.unhexlify(key)

    return hmac.new(bin_key, bin_counter, hash).digest()

def hotp(key,counter,format='dec6',hash=hashlib.sha1):
    '''Compute a HOTP value as prescribed by RFC4226

       See http://tools.ietf.org/html/rfc4226
    '''
    bin_hotp = __hotp(key, counter, hash)

    if format == 'hex40':
        return binascii.hexlify(bin_hotp[0:5])
    elif format == 'dec4':
        return dec(bin_hotp, 4)
    elif format == 'dec6':
        return dec(bin_hotp, 6)
    elif format == 'dec7':
        return dec(bin_hotp, 7)
    elif format == 'dec8':
        return dec(bin_hotp, 8)
    else:
        raise ValueError('unknown format')

def accept_hotp(key, response, counter, format='dec6', hash=hashlib.sha1,
        drift=3, backward_drift=0):
    '''
        Validate an HOTP value inside a window of
        [counter-backward_drift:counter+forward_drift]

        :params key:
            the shared secret
        :type key:
            hexadecimal string of even length
        :params response:
            the OTP to check
        :type response:
            ASCII string
        :params counter:
            value of the counter running inside an HOTP token, usually it is
            just the count of HOTP value accepte so far for a given shared
            secret
        :params format:
            the format of the HOTP hash to generate
        :params hash:
            the hashing algorithm to use, default to SHA1
        :params drift:
            how far we can look forward from the current value of the counter
        :params backward_drift:
            how far we can look backward from the current counter value to
            match the response, default to zero as it is usually a bad idea to
            look backward as the counter is only advanced when a valid value is
            checked (and so the counter on the token side should have been
            incremented too)

        :returns:
            a pair of a boolean and an integer:
             - first is True if the response is validated and False otherwise,
             - second is the new value for the counter; it can be more than
               counter + 1 if the drift window was used; you must store it if
               the response was validated.

        >>> accept_hotp('343434', '122323', 2, format='dec6')
            (False, 2)

        >>> hotp('343434', 2, format='dec6')
            '791903'

        >>> accept_hotp('343434', '791903', 2, format='dec6')
            (True, 3)

        >>> hotp('343434', 3, format='dec6')
            '907279'

        >>> accept_hotp('343434', '907279', 2, format='dec6')
            (True, 4)
    '''

    for i in range(-backward_drift, drift+1):
        if hotp(key, counter+i, format=format, hash=hash) == response:
            return True, counter+i+1
    return False,counter

if __name__ == '__main__':
    # Test vectors extracted from RFC 4226
    secret = '3132333435363738393031323334353637383930'
    tvector = [
        (0, 'cc93cf18508d94934c64b65d8ba7667fb7cde4b0'),
        (1, '75a48a19d4cbe100644e8ac1397eea747a2d33ab'),
        (2, '0bacb7fa082fef30782211938bc1c5e70416ff44'),
        (3, '66c28227d03a2d5529262ff016a1e6ef76557ece'),
        (4, 'a904c900a64b35909874b33e61c5938a8e15ed1c'),
        (5, 'a37e783d7b7233c083d4f62926c7a25f238d0316'),
        (6, 'bc9cd28561042c83f219324d3c607256c03272ae'),
        (7, 'a4fb960c0bc06e1eabb804e5b397cdc4b45596fa'),
        (8, '1b3c89f65e6c9e883012052823443f048b4332db'),
        (9, '1637409809a679dc698207310c8c7fc07290d9e5'), ]
    for counter, value in tvector:
        assert(binascii.hexlify(__hotp(secret, counter)) == value)
    tvector2 = [
        (0, '4c93cf18', '1284755224', '755224',),
        (1, '41397eea', '1094287082', '287082',),
        (2, '82fef30',  '137359152',  '359152',),
        (3, '66ef7655', '1726969429', '969429',),
        (4, '61c5938a', '1640338314', '338314',),
        (5, '33c083d4', '868254676',  '254676',),
        (6, '7256c032', '1918287922', '287922',),
        (7, '4e5b397',  '82162583',   '162583',),
        (8, '2823443f', '673399871',  '399871',),
        (9, '2679dc69',  '645520489', '520489',),]
    for counter, hexa, deci, trunc in tvector2:
        h = __hotp(secret, counter)
        v = __truncated_value(h)
        assert(hex(v)[2:] == hexa)
        assert(str(v) == deci)
        assert(dec(h,6) == trunc)
        assert accept_hotp(secret, trunc, counter)
<commit_msg>Factorize bigendian int64 formatting, and publicize truncation function<commit_after>import hashlib
import hmac
import binascii

'''
Python implementation of HOTP and TOTP algorithms from the OATH project.
'''

def truncated_value(h):
    bytes = map(ord, h)
    offset = bytes[-1] & 0xf
    v = (bytes[offset] & 0x7f) << 24 | (bytes[offset+1] & 0xff) << 16 | \
            (bytes[offset+2] & 0xff) << 8 | (bytes[offset+3] & 0xff)
    return v

def dec(h,p):
    v = str(truncated_value(h))
    return v[len(v)-p:]

def int2beint64(i):
    hex_counter = hex(long(i))[2:-1]
    hex_counter = '0' * (16 - len(hex_counter)) + hex_counter
    bin_counter = binascii.unhexlify(hex_counter)
    return bin_counter

def __hotp(key, counter, hash=hashlib.sha1):
    bin_counter = int2beint64(counter)
    bin_key = binascii.unhexlify(key)

    return hmac.new(bin_key, bin_counter, hash).digest()

def hotp(key,counter,format='dec6',hash=hashlib.sha1):
    '''Compute a HOTP value as prescribed by RFC4226

       See http://tools.ietf.org/html/rfc4226
    '''
    bin_hotp = __hotp(key, counter, hash)

    if format == 'hex40':
        return binascii.hexlify(bin_hotp[0:5])
    elif format == 'dec4':
        return dec(bin_hotp, 4)
    elif format == 'dec6':
        return dec(bin_hotp, 6)
    elif format == 'dec7':
        return dec(bin_hotp, 7)
    elif format == 'dec8':
        return dec(bin_hotp, 8)
    else:
        raise ValueError('unknown format')

def accept_hotp(key, response, counter, format='dec6', hash=hashlib.sha1,
        drift=3, backward_drift=0):
    '''
        Validate an HOTP value inside a window of
        [counter-backward_drift:counter+forward_drift]

        :params key:
            the shared secret
        :type key:
            hexadecimal string of even length
        :params response:
            the OTP to check
        :type response:
            ASCII string
        :params counter:
            value of the counter running inside an HOTP token, usually it is
            just the count of HOTP value accepte so far for a given shared
            secret
        :params format:
            the format of the HOTP hash to generate
        :params hash:
            the hashing algorithm to use, default to SHA1
        :params drift:
            how far we can look forward from the current value of the counter
        :params backward_drift:
            how far we can look backward from the current counter value to
            match the response, default to zero as it is usually a bad idea to
            look backward as the counter is only advanced when a valid value is
            checked (and so the counter on the token side should have been
            incremented too)

        :returns:
            a pair of a boolean and an integer:
             - first is True if the response is validated and False otherwise,
             - second is the new value for the counter; it can be more than
               counter + 1 if the drift window was used; you must store it if
               the response was validated.

        >>> accept_hotp('343434', '122323', 2, format='dec6')
            (False, 2)

        >>> hotp('343434', 2, format='dec6')
            '791903'

        >>> accept_hotp('343434', '791903', 2, format='dec6')
            (True, 3)

        >>> hotp('343434', 3, format='dec6')
            '907279'

        >>> accept_hotp('343434', '907279', 2, format='dec6')
            (True, 4)
    '''

    for i in range(-backward_drift, drift+1):
        if hotp(key, counter+i, format=format, hash=hash) == response:
            return True, counter+i+1
    return False,counter

if __name__ == '__main__':
    # Test vectors extracted from RFC 4226
    secret = '3132333435363738393031323334353637383930'
    tvector = [
        (0, 'cc93cf18508d94934c64b65d8ba7667fb7cde4b0'),
        (1, '75a48a19d4cbe100644e8ac1397eea747a2d33ab'),
        (2, '0bacb7fa082fef30782211938bc1c5e70416ff44'),
        (3, '66c28227d03a2d5529262ff016a1e6ef76557ece'),
        (4, 'a904c900a64b35909874b33e61c5938a8e15ed1c'),
        (5, 'a37e783d7b7233c083d4f62926c7a25f238d0316'),
        (6, 'bc9cd28561042c83f219324d3c607256c03272ae'),
        (7, 'a4fb960c0bc06e1eabb804e5b397cdc4b45596fa'),
        (8, '1b3c89f65e6c9e883012052823443f048b4332db'),
        (9, '1637409809a679dc698207310c8c7fc07290d9e5'), ]
    for counter, value in tvector:
        assert(binascii.hexlify(__hotp(secret, counter)) == value)
    tvector2 = [
        (0, '4c93cf18', '1284755224', '755224',),
        (1, '41397eea', '1094287082', '287082',),
        (2, '82fef30',  '137359152',  '359152',),
        (3, '66ef7655', '1726969429', '969429',),
        (4, '61c5938a', '1640338314', '338314',),
        (5, '33c083d4', '868254676',  '254676',),
        (6, '7256c032', '1918287922', '287922',),
        (7, '4e5b397',  '82162583',   '162583',),
        (8, '2823443f', '673399871',  '399871',),
        (9, '2679dc69',  '645520489', '520489',),]
    for counter, hexa, deci, trunc in tvector2:
        h = __hotp(secret, counter)
        v = truncated_value(h)
        assert(hex(v)[2:] == hexa)
        assert(str(v) == deci)
        assert(dec(h,6) == trunc)
        assert accept_hotp(secret, trunc, counter)
<|endoftext|>


<commit_before>from __future__ import print_function

import os.path
import json

try:
    from matplotlib.colors import LinearSegmentedColormap
except ImportError:
    HAVE_MPL = False
else:
    HAVE_MPL = True

__all__ = ('COLOR_MAPS', 'print_maps', 'print_all_maps', 'print_maps_by_type',
           'get_map', 'MAP_TYPES', 'BrewerMap')

_DATADIR = os.path.join(os.path.dirname(__file__), 'data')
_DATAFILE = os.path.join(_DATADIR, 'colorbrewer_all_schemes.json')

with open(_DATAFILE, 'r') as f:
    COLOR_MAPS = json.load(f)

MAP_TYPES = ('Sequential', 'Diverging', 'Qualitative')

def print_maps(map_type=None, number=None):
    """
    Print maps by type and/or number of defined colors.

    Parameters
    ----------
    map_type : {'Sequential', 'Diverging', 'Qualitative'}, optional
        Filter output by map type. By default all maps are printed.
    number : int, optional
        Filter output by number of defined colors. By default there is
        no numeric filtering.

    """
    if not map_type and not number:
        print_all_maps()

    elif map_type:
        print_maps_by_type(map_type, number)

    else:
        s = ('Invalid parameter combination. '
             'number without map_type is not supported.')
        raise ValueError(s)

def print_all_maps():
    """
    Print the name and number of defined colors of all available color maps.

    """
    for t in MAP_TYPES:
        print_maps_by_type(t)

def print_maps_by_type(map_type, number=None):
    """
    Print all available maps of a given type.

    Parameters
    ----------
    map_type : {'Sequential', 'Diverging', 'Qualitative'}
        Select map type to print.
    number : int, optional
        Filter output by number of defined colors. By default there is
        no numeric filtering.

    """
    map_type = map_type.capitalize()
    if map_type not in MAP_TYPES:
        s = 'Invalid map type, must be one of {}'.format(MAP_TYPES)
        raise ValueError(s)

    print(map_type)

    map_keys = sorted(COLOR_MAPS[map_type].keys())

    format_str = '{0:8}  :  {1}'

    for mk in map_keys:
        num_keys = sorted(COLOR_MAPS[map_type][mk].keys(), key=int)

        if not number or str(number) in num_keys:
            num_str = '{' + ', '.join(num_keys) + '}'
            print(format_str.format(mk, num_str))

class BrewerMap(object):
    """
    Representation of a colorbrewer2 color map with matplotlib compatible
    views of the map.

    Parameters
    ----------
    name : str
    map_type : str
    colors : list
        Colors as list of 0-255 RGB triplets.

    Attributes
    ----------
    name : str
    map_type : str
    number : int
        Number of colors in color map.
    colors : list
        Colors as list of 0-255 RGB triplets.
    colorbrewer2_url : str
    hex_colors : list
    mpl_colors : list
    mpl_colormap : matplotlib LinearSegmentedColormap

    """
    def __init__(self, name, map_type, colors):
        self.name = name
        self.type = map_type
        self.number = len(colors)
        self.colors = colors

    @property
    def colorbrewer2_url(self):
        """
        URL that can be used to view the color map at colorbrewer2.org.

        """
        url = 'http://colorbrewer2.org/index.php?type={0}&scheme={1}&n={2}'
        return url.format(self.type.lower(), self.name, self.number)

    @property
    def hex_colors(self):
        """
        Colors as a list of hex strings. (e.g. '#A912F4')

        """
        hc = []

        for color in self.colors:
            h = '#' + ''.join('{:>02}'.format(hex(c)[2:].upper()) for c in color)
            hc.append(h)

        return hc

    @property
    def mpl_colors(self):
        """
        Colors expressed on the range 0-1 as used by matplotlib.

        """
        mc = []

        for color in self.colors:
            mc.append(tuple([x/255. for x in color]))

        return mc

    @property
    def mpl_colormap(self):
        """
        A basic matplotlib color map. If you want to specify keyword arguments
        use the `get_mpl_colormap` method.

        """
        return self.get_mpl_colormap()

    def get_mpl_colormap(self, **kwargs):
        """
        A color map that can be used in matplotlib plots. Requires matplotlib
        to be importable. Keyword arguments are passed to
        `matplotlib.colors.LinearSegmentedColormap.from_list`.

        """
        if not HAVE_MPL:
            raise RuntimeError('matplotlib not available.')

        cmap = LinearSegmentedColormap.from_list(self.name,
                                                 self.mpl_colors, **kwargs)

        return cmap

def get_map(name, map_type, number, reverse=False):
    """
    Return a `BrewerMap` representation of the specified color map.

    Parameters
    ----------
    name : str
        Name of color map. The name is case sensitive, use `print_maps`
        to see available color maps.
    map_type : {'Sequential', 'Diverging', 'Qualitative'}
        Select color map type.
    number : int
        Number of defined colors in color map.
    reverse : bool, optional
        Set to True to get the reversed color map.

    """
    number = str(number)
    map_type = map_type.capitalize()

    colors = COLOR_MAPS[map_type][name][number]['Colors']

    if reverse:
        name += '_r'
        colors.reverse()

    return BrewerMap(name, map_type, int(number), colors)
<commit_msg>Fixed call to BrewerMap constructor. get_map was still passing the number parameter.<commit_after>from __future__ import print_function

import os.path
import json

try:
    from matplotlib.colors import LinearSegmentedColormap
except ImportError:
    HAVE_MPL = False
else:
    HAVE_MPL = True

__all__ = ('COLOR_MAPS', 'print_maps', 'print_all_maps', 'print_maps_by_type',
           'get_map', 'MAP_TYPES', 'BrewerMap')

_DATADIR = os.path.join(os.path.dirname(__file__), 'data')
_DATAFILE = os.path.join(_DATADIR, 'colorbrewer_all_schemes.json')

with open(_DATAFILE, 'r') as f:
    COLOR_MAPS = json.load(f)

MAP_TYPES = ('Sequential', 'Diverging', 'Qualitative')

def print_maps(map_type=None, number=None):
    """
    Print maps by type and/or number of defined colors.

    Parameters
    ----------
    map_type : {'Sequential', 'Diverging', 'Qualitative'}, optional
        Filter output by map type. By default all maps are printed.
    number : int, optional
        Filter output by number of defined colors. By default there is
        no numeric filtering.

    """
    if not map_type and not number:
        print_all_maps()

    elif map_type:
        print_maps_by_type(map_type, number)

    else:
        s = ('Invalid parameter combination. '
             'number without map_type is not supported.')
        raise ValueError(s)

def print_all_maps():
    """
    Print the name and number of defined colors of all available color maps.

    """
    for t in MAP_TYPES:
        print_maps_by_type(t)

def print_maps_by_type(map_type, number=None):
    """
    Print all available maps of a given type.

    Parameters
    ----------
    map_type : {'Sequential', 'Diverging', 'Qualitative'}
        Select map type to print.
    number : int, optional
        Filter output by number of defined colors. By default there is
        no numeric filtering.

    """
    map_type = map_type.capitalize()
    if map_type not in MAP_TYPES:
        s = 'Invalid map type, must be one of {}'.format(MAP_TYPES)
        raise ValueError(s)

    print(map_type)

    map_keys = sorted(COLOR_MAPS[map_type].keys())

    format_str = '{0:8}  :  {1}'

    for mk in map_keys:
        num_keys = sorted(COLOR_MAPS[map_type][mk].keys(), key=int)

        if not number or str(number) in num_keys:
            num_str = '{' + ', '.join(num_keys) + '}'
            print(format_str.format(mk, num_str))

class BrewerMap(object):
    """
    Representation of a colorbrewer2 color map with matplotlib compatible
    views of the map.

    Parameters
    ----------
    name : str
    map_type : str
    colors : list
        Colors as list of 0-255 RGB triplets.

    Attributes
    ----------
    name : str
    map_type : str
    number : int
        Number of colors in color map.
    colors : list
        Colors as list of 0-255 RGB triplets.
    colorbrewer2_url : str
    hex_colors : list
    mpl_colors : list
    mpl_colormap : matplotlib LinearSegmentedColormap

    """
    def __init__(self, name, map_type, colors):
        self.name = name
        self.type = map_type
        self.number = len(colors)
        self.colors = colors

    @property
    def colorbrewer2_url(self):
        """
        URL that can be used to view the color map at colorbrewer2.org.

        """
        url = 'http://colorbrewer2.org/index.php?type={0}&scheme={1}&n={2}'
        return url.format(self.type.lower(), self.name, self.number)

    @property
    def hex_colors(self):
        """
        Colors as a list of hex strings. (e.g. '#A912F4')

        """
        hc = []

        for color in self.colors:
            h = '#' + ''.join('{:>02}'.format(hex(c)[2:].upper()) for c in color)
            hc.append(h)

        return hc

    @property
    def mpl_colors(self):
        """
        Colors expressed on the range 0-1 as used by matplotlib.

        """
        mc = []

        for color in self.colors:
            mc.append(tuple([x/255. for x in color]))

        return mc

    @property
    def mpl_colormap(self):
        """
        A basic matplotlib color map. If you want to specify keyword arguments
        use the `get_mpl_colormap` method.

        """
        return self.get_mpl_colormap()

    def get_mpl_colormap(self, **kwargs):
        """
        A color map that can be used in matplotlib plots. Requires matplotlib
        to be importable. Keyword arguments are passed to
        `matplotlib.colors.LinearSegmentedColormap.from_list`.

        """
        if not HAVE_MPL:
            raise RuntimeError('matplotlib not available.')

        cmap = LinearSegmentedColormap.from_list(self.name,
                                                 self.mpl_colors, **kwargs)

        return cmap

def get_map(name, map_type, number, reverse=False):
    """
    Return a `BrewerMap` representation of the specified color map.

    Parameters
    ----------
    name : str
        Name of color map. The name is case sensitive, use `print_maps`
        to see available color maps.
    map_type : {'Sequential', 'Diverging', 'Qualitative'}
        Select color map type.
    number : int
        Number of defined colors in color map.
    reverse : bool, optional
        Set to True to get the reversed color map.

    """
    number = str(number)
    map_type = map_type.capitalize()

    colors = COLOR_MAPS[map_type][name][number]['Colors']

    if reverse:
        name += '_r'
        colors.reverse()

    return BrewerMap(name, map_type, colors)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# RHEAS documentation build configuration file, created by
# sphinx-quickstart on Sat Jan 24 16:09:21 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../src'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.2.3'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.pngmath',
    'sphinx.ext.viewcode',
    'sphinx.ext.graphviz',
    'sphinx.ext.inheritance_diagram'
    # 'sphinxcontrib.programoutput',
    # 'sphinxcontrib.plantuml'
]

# Set PlantUML path
# plantuml = 'java -jar /usr/local/Cellar/plantuml/8018/plantuml.8018.jar'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'RHEAS'
copyright = u'2015, Kostas Andreadis'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.2'
# The full version, including alpha/beta/rc tags.
release = '0.2'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
html_domain_indices = True

# If false, no index is generated.
html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'RHEASdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'RHEAS.tex', u'RHEAS Documentation',
   u'Kostas Andreadis', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'rheas', u'RHEAS Documentation',
     [u'Kostas Andreadis'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'RHEAS', u'RHEAS Documentation',
   u'Kostas Andreadis', 'RHEAS', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
<commit_msg>Added mock modules in documentation configuration.<commit_after># -*- coding: utf-8 -*-
#
# RHEAS documentation build configuration file, created by
# sphinx-quickstart on Sat Jan 24 16:09:21 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# Mock out the imports for some modules
from mock import MagicMock

class Mock(MagicMock):
    @classmethod
    def __getattr__(cls, name):
            return Mock()

MOCK_MODULES = ['argparse', 'numpy', 'pandas', 'psycopg2', 'scipy', 'scipy.linalg', 'netCDF4',
                'scipy.stats', 'scipy.spatial', 'scipy.spatial.distance', 'GDAL', 'lxml', 'lxml.html',
                'requests', 'h5py', 'osgeo', 'python-dateutil', 'dateutil', 'dateutil.relativedelta', 'requests.auth']
sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../src'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.2.3'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.pngmath',
    'sphinx.ext.viewcode',
    'sphinx.ext.graphviz',
    'sphinx.ext.inheritance_diagram'
    # 'sphinxcontrib.programoutput',
    # 'sphinxcontrib.plantuml'
]

# Set PlantUML path
# plantuml = 'java -jar /usr/local/Cellar/plantuml/8018/plantuml.8018.jar'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'RHEAS'
copyright = u'2015, Kostas Andreadis'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.2'
# The full version, including alpha/beta/rc tags.
release = '0.2'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
html_domain_indices = True

# If false, no index is generated.
html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'RHEASdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'RHEAS.tex', u'RHEAS Documentation',
   u'Kostas Andreadis', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'rheas', u'RHEAS Documentation',
     [u'Kostas Andreadis'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'RHEAS', u'RHEAS Documentation',
   u'Kostas Andreadis', 'RHEAS', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
<|endoftext|>


<commit_before>import os
import json
import pprint
import humanize
import functools

from datetime import datetime, timedelta
from holster.emitter import Priority
from disco.api.http import APIException
from disco.bot.command import CommandEvent, CommandLevels

from rowboat import VERSION
from rowboat.plugins import BasePlugin as Plugin
from rowboat.plugins import RowboatPlugin
from rowboat.sql import init_db
from rowboat.redis import rdb
from rowboat.models.guild import Guild
from rowboat.models.notification import Notification
from rowboat.plugins.modlog import Actions

ENV = os.getenv('ENV', 'local')

PY_CODE_BLOCK = u'```py\n{}\n```'

INFO_MESSAGE = '''\
:information_source: Rowboat V{} - more information and detailed help can be found here:\
<https://github.com/b1naryth1ef/rowboat/wiki>
'''.format(VERSION)

class CorePlugin(Plugin):
    def load(self, ctx):
        init_db()

        self.startup = ctx.get('startup', datetime.utcnow())
        self.guilds = ctx.get('guilds', {})

        super(CorePlugin, self).load(ctx)

        for plugin in self.bot.plugins.values():
            if not isinstance(plugin, RowboatPlugin):
                continue

            plugin.register_trigger('command', 'pre', functools.partial(self.on_pre, plugin))
            plugin.register_trigger('listener', 'pre', functools.partial(self.on_pre, plugin))

        self.spawn(self.wait_for_updates)
        self.spawn(self.wait_for_dispatches)

    def wait_for_dispatches(self):
        ps = rdb.pubsub()
        ps.subscribe('notifications')

        for item in ps.listen():
            if item['type'] != 'message':
                continue

            obj = json.loads(item['data'])

            self.bot.client.api.channels_messages_create(
                290924692057882635,
                u'**{}**\n{}'.format(
                    obj['title'],
                    obj['content']))

    def wait_for_updates(self):
        ps = rdb.pubsub()
        ps.subscribe('guild-updates')

        for item in ps.listen():
            if item['type'] != 'message':
                continue

            data = json.loads(item['data'])
            if data['type'] == 'UPDATE' and data['id'] in self.guilds:
                self.send_control_message(u'Reloaded config for Guild {}'.format(self.guilds[data['id']].name))
                self.log.info('Reloading config for guild %s', self.guilds[data['id']].name)
                self.guilds[data['id']].get_config(refresh=True)

    def unload(self, ctx):
        ctx['guilds'] = self.guilds
        ctx['startup'] = self.startup
        super(CorePlugin, self).unload(ctx)

    def on_pre(self, plugin, func, event, args, kwargs):
        """
        This function handles dynamically dispatching and modifying events based
        on a specific guilds configuration. It is called before any handler of
        either commands or listeners.
        """
        if isinstance(event, CommandEvent):
            if event.command.metadata.get('global_', False):
                return event
        elif hasattr(func, 'subscriptions'):
            if func.subscriptions[0].metadata.get('global_', False):
                return event

        if hasattr(event, 'guild') and event.guild:
            guild_id = event.guild.id
        elif hasattr(event, 'guild_id') and event.guild_id:
            guild_id = event.guild_id
        else:
            return

        if guild_id not in self.guilds:
            return

        event.base_config = self.guilds[guild_id].get_config()

        plugin_name = plugin.name.lower().replace('plugin', '')
        if not getattr(event.base_config.plugins, plugin_name, None):
            return

        event.config = getattr(event.base_config.plugins, plugin_name)
        return event

    @Plugin.schedule(290, init=False)
    def update_guild_bans(self):
        to_update = [
            guild for guild in Guild.select().where(
                (Guild.last_ban_sync < (datetime.utcnow() - timedelta(days=1))) |
                (Guild.last_ban_sync >> None)
            )
            if guild.guild_id in self.client.state.guilds]

        # Update 10 at a time
        for guild in to_update[:10]:
            guild.sync_bans(self.client.state.guilds.get(guild.guild_id))

    def send_control_message(self, content, *args, **kwargs):
        self.bot.client.api.channels_messages_create(
            290924692057882635,
            u'**{}**\n{}'.format('Production' if ENV == 'prod' else 'Local', content), *args, **kwargs)

    @Plugin.listen('Resumed')
    def on_resumed(self, event):
        Notification.dispatch(
            Notification.Types.RESUME,
            trace=event.trace,
            env=ENV,
        )

    @Plugin.listen('Ready')
    def on_ready(self, event):
        Notification.dispatch(
            Notification.Types.CONNECT,
            trace=event.trace,
            env=ENV,
        )

    @Plugin.listen('GuildCreate', conditional=lambda e: e.created is True)
    def on_guild_join(self, event):
        self.send_control_message('Added to guild {}: {}'.format(event.guild.id, event.guild.name))

    @Plugin.listen('GuildDelete', conditional=lambda e: e.deleted is True)
    def on_guild_leave(self, event):
        self.send_control_message('Removed from guild {}'.format(event.guild.id))

    @Plugin.listen('GuildCreate', priority=Priority.BEFORE, conditional=lambda e: not e.created)
    def on_guild_create(self, event):
        try:
            guild = Guild.with_id(event.id)
        except Guild.DoesNotExist:
            return

        if not guild.enabled:
            return

        # Ensure we're updated
        guild.sync(event.guild)

        self.guilds[event.id] = guild

        if guild.get_config().nickname:
            def set_nickname():
                m = event.members.select_one(id=self.state.me.id)
                if m and m.nick != guild.get_config().nickname:
                    try:
                        m.set_nickname(guild.get_config().nickname)
                    except APIException as e:
                        self.log.warning('Failed to set nickname for guild %s (%s)', event.guild, e.content)
            self.spawn_later(5, set_nickname)

    def get_level(self, guild, user):
        config = (guild.id in self.guilds and self.guilds.get(guild.id).get_config())

        user_level = 0
        if config:
            for oid in guild.get_member(user.id).roles:
                if oid in config.levels and config.levels[oid] > user_level:
                    user_level = config.levels[oid]

            # User ID overrides should override all others
            if user.id in config.levels:
                user_level = config.levels[user.id]

        return user_level

    @Plugin.listen('MessageCreate')
    def on_message_create(self, event):
        """
        This monstrosity of a function handles the parsing and dispatching of
        commands.
        """
        # Ignore messages sent by bots
        if event.message.author.bot:
            return

        if rdb.sismember('ignored_channels', event.message.channel_id):
            return

        # If this is message for a guild, grab the guild object
        if hasattr(event, 'guild') and event.guild:
            guild_id = event.guild.id
        elif hasattr(event, 'guild_id') and event.guild_id:
            guild_id = event.guild_id
        else:
            guild_id = None

        guild = self.guilds.get(event.guild.id) if guild_id else None
        config = guild and guild.get_config()

        # If the guild has configuration, use that (otherwise use defaults)
        if config:
            if config.commands:
                commands = list(self.bot.get_commands_for_message(
                    config.commands.mention,
                    {},
                    config.commands.prefix,
                    event.message))
        elif guild_id:
            # Otherwise, default to requiring mentions
            commands = list(self.bot.get_commands_for_message(True, {}, '', event.message))
        else:
            if ENV != 'prod':
                if not event.message.content.startswith(ENV + '!'):
                    return
                event.message.content = event.message.content[len(ENV) + 1:]

            # DM's just use the commands (no prefix/mention)
            commands = list(self.bot.get_commands_for_message(False, {}, '', event.message))

        # If we didn't find any matching commands, return
        if not len(commands):
            return

        user_level = self.get_level(event.guild, event.author)

        # Grab whether this user is a global admin
        # TODO: cache this
        global_admin = rdb.sismember('global_admins', event.author.id)

        # Iterate over commands and find a match
        for command, match in commands:
            if command.level == -1 and not global_admin:
                continue

            level = command.level

            if guild and not config and command.triggers[0] != 'setup':
                continue
            elif config and config.commands and command.plugin != self:
                if command.triggers[0] in config.commands.overrides or '*' in config.commands.overrides:
                    override = config.commands.overrides.get(
                        command.triggers[0],
                        config.commands.overrides.get('*'))
                    if override.disabled:
                        continue

                    if override.level is not None:
                        level = override.level

            if not global_admin and user_level < level:
                continue

            command.plugin.execute(CommandEvent(command, event.message, match))

            # Dispatch the command used modlog event
            if config:
                event.config = getattr(config.plugins, 'modlog', None)
                if not event.config:
                    return

                plugin = self.bot.plugins.get('ModLogPlugin')
                if plugin:
                    plugin.log_action(Actions.COMMAND_USED, event)

            return

    @Plugin.command('setup')
    def command_setup(self, event):
        """
        Setup a new Guild with Rowboat
        """
        if not event.guild:
            return event.msg.reply(':warning: this command can only be used in servers')

        # Make sure we're not already setup
        if event.guild.id in self.guilds:
            return event.msg.reply(':warning: this server is already setup')

        global_admin = rdb.sismember('global_admins', event.author.id)

        # Make sure this is the owner of the server
        if not global_admin:
            if not event.guild.owner_id == event.author.id:
                return event.msg.reply(':warning: only the server owner can setup rowboat')

        # Make sure we have admin perms
        m = event.guild.members.select_one(id=self.state.me.id)
        if not m.permissions.administrator and not global_admin:
            return event.msg.reply(':warning: bot must have the Administrator permission')

        guild = Guild.setup(event.guild)
        self.guilds[event.guild.id] = guild
        event.msg.reply(':ok_hand: successfully loaded configuration')

    @Plugin.command('about', level=CommandLevels.ADMIN)
    def command_help(self, event):
        event.msg.reply(INFO_MESSAGE)

    @Plugin.command('uptime', level=-1)
    def command_uptime(self, event):
        event.msg.reply('Rowboat was started {}'.format(
            humanize.naturaltime(datetime.utcnow() - self.startup)
        ))

    @Plugin.command('eval', level=-1)
    def command_eval(self, event):
        ctx = {
            'bot': self.bot,
            'client': self.bot.client,
            'state': self.bot.client.state,
            'event': event,
            'msg': event.msg,
            'guild': event.msg.guild,
            'channel': event.msg.channel,
            'author': event.msg.author
        }

        # Mulitline eval
        src = event.codeblock
        if src.count('\n'):
            lines = filter(bool, src.split('\n'))
            if lines[-1] and 'return' not in lines[-1]:
                lines[-1] = 'return ' + lines[-1]
            lines = '\n'.join('    ' + i for i in lines)
            code = 'def f():\n{}\nx = f()'.format(lines)
            local = {}

            try:
                exec compile(code, '<eval>', 'exec') in ctx, local
            except Exception as e:
                event.msg.reply(PY_CODE_BLOCK.format(type(e).__name__ + ': ' + str(e)))
                return

            event.msg.reply(PY_CODE_BLOCK.format(pprint.pformat(local['x'])))
        else:
            try:
                result = eval(src, ctx)
            except Exception as e:
                event.msg.reply(PY_CODE_BLOCK.format(type(e).__name__ + ': ' + str(e)))
                return

            event.msg.reply(PY_CODE_BLOCK.format(result))
<commit_msg>Fix grabbing user level while still syncing<commit_after>import os
import json
import pprint
import humanize
import functools

from datetime import datetime, timedelta
from holster.emitter import Priority
from disco.api.http import APIException
from disco.bot.command import CommandEvent, CommandLevels

from rowboat import VERSION
from rowboat.plugins import BasePlugin as Plugin
from rowboat.plugins import RowboatPlugin
from rowboat.sql import init_db
from rowboat.redis import rdb
from rowboat.models.guild import Guild
from rowboat.models.notification import Notification
from rowboat.plugins.modlog import Actions

ENV = os.getenv('ENV', 'local')

PY_CODE_BLOCK = u'```py\n{}\n```'

INFO_MESSAGE = '''\
:information_source: Rowboat V{} - more information and detailed help can be found here:\
<https://github.com/b1naryth1ef/rowboat/wiki>
'''.format(VERSION)

class CorePlugin(Plugin):
    def load(self, ctx):
        init_db()

        self.startup = ctx.get('startup', datetime.utcnow())
        self.guilds = ctx.get('guilds', {})

        super(CorePlugin, self).load(ctx)

        for plugin in self.bot.plugins.values():
            if not isinstance(plugin, RowboatPlugin):
                continue

            plugin.register_trigger('command', 'pre', functools.partial(self.on_pre, plugin))
            plugin.register_trigger('listener', 'pre', functools.partial(self.on_pre, plugin))

        self.spawn(self.wait_for_updates)
        self.spawn(self.wait_for_dispatches)

    def wait_for_dispatches(self):
        ps = rdb.pubsub()
        ps.subscribe('notifications')

        for item in ps.listen():
            if item['type'] != 'message':
                continue

            obj = json.loads(item['data'])

            self.bot.client.api.channels_messages_create(
                290924692057882635,
                u'**{}**\n{}'.format(
                    obj['title'],
                    obj['content']))

    def wait_for_updates(self):
        ps = rdb.pubsub()
        ps.subscribe('guild-updates')

        for item in ps.listen():
            if item['type'] != 'message':
                continue

            data = json.loads(item['data'])
            if data['type'] == 'UPDATE' and data['id'] in self.guilds:
                self.send_control_message(u'Reloaded config for Guild {}'.format(self.guilds[data['id']].name))
                self.log.info('Reloading config for guild %s', self.guilds[data['id']].name)
                self.guilds[data['id']].get_config(refresh=True)

    def unload(self, ctx):
        ctx['guilds'] = self.guilds
        ctx['startup'] = self.startup
        super(CorePlugin, self).unload(ctx)

    def on_pre(self, plugin, func, event, args, kwargs):
        """
        This function handles dynamically dispatching and modifying events based
        on a specific guilds configuration. It is called before any handler of
        either commands or listeners.
        """
        if isinstance(event, CommandEvent):
            if event.command.metadata.get('global_', False):
                return event
        elif hasattr(func, 'subscriptions'):
            if func.subscriptions[0].metadata.get('global_', False):
                return event

        if hasattr(event, 'guild') and event.guild:
            guild_id = event.guild.id
        elif hasattr(event, 'guild_id') and event.guild_id:
            guild_id = event.guild_id
        else:
            return

        if guild_id not in self.guilds:
            return

        event.base_config = self.guilds[guild_id].get_config()

        plugin_name = plugin.name.lower().replace('plugin', '')
        if not getattr(event.base_config.plugins, plugin_name, None):
            return

        event.config = getattr(event.base_config.plugins, plugin_name)
        return event

    @Plugin.schedule(290, init=False)
    def update_guild_bans(self):
        to_update = [
            guild for guild in Guild.select().where(
                (Guild.last_ban_sync < (datetime.utcnow() - timedelta(days=1))) |
                (Guild.last_ban_sync >> None)
            )
            if guild.guild_id in self.client.state.guilds]

        # Update 10 at a time
        for guild in to_update[:10]:
            guild.sync_bans(self.client.state.guilds.get(guild.guild_id))

    def send_control_message(self, content, *args, **kwargs):
        self.bot.client.api.channels_messages_create(
            290924692057882635,
            u'**{}**\n{}'.format('Production' if ENV == 'prod' else 'Local', content), *args, **kwargs)

    @Plugin.listen('Resumed')
    def on_resumed(self, event):
        Notification.dispatch(
            Notification.Types.RESUME,
            trace=event.trace,
            env=ENV,
        )

    @Plugin.listen('Ready')
    def on_ready(self, event):
        Notification.dispatch(
            Notification.Types.CONNECT,
            trace=event.trace,
            env=ENV,
        )

    @Plugin.listen('GuildCreate', conditional=lambda e: e.created is True)
    def on_guild_join(self, event):
        self.send_control_message('Added to guild {}: {}'.format(event.guild.id, event.guild.name))

    @Plugin.listen('GuildDelete', conditional=lambda e: e.deleted is True)
    def on_guild_leave(self, event):
        self.send_control_message('Removed from guild {}'.format(event.guild.id))

    @Plugin.listen('GuildCreate', priority=Priority.BEFORE, conditional=lambda e: not e.created)
    def on_guild_create(self, event):
        try:
            guild = Guild.with_id(event.id)
        except Guild.DoesNotExist:
            return

        if not guild.enabled:
            return

        # Ensure we're updated
        guild.sync(event.guild)

        self.guilds[event.id] = guild

        if guild.get_config().nickname:
            def set_nickname():
                m = event.members.select_one(id=self.state.me.id)
                if m and m.nick != guild.get_config().nickname:
                    try:
                        m.set_nickname(guild.get_config().nickname)
                    except APIException as e:
                        self.log.warning('Failed to set nickname for guild %s (%s)', event.guild, e.content)
            self.spawn_later(5, set_nickname)

    def get_level(self, guild, user):
        config = (guild.id in self.guilds and self.guilds.get(guild.id).get_config())

        user_level = 0
        if config:
            member = guild.get_member(user.id)
            if not member:
                return user_level

            for oid in member.roles:
                if oid in config.levels and config.levels[oid] > user_level:
                    user_level = config.levels[oid]

            # User ID overrides should override all others
            if user.id in config.levels:
                user_level = config.levels[user.id]

        return user_level

    @Plugin.listen('MessageCreate')
    def on_message_create(self, event):
        """
        This monstrosity of a function handles the parsing and dispatching of
        commands.
        """
        # Ignore messages sent by bots
        if event.message.author.bot:
            return

        if rdb.sismember('ignored_channels', event.message.channel_id):
            return

        # If this is message for a guild, grab the guild object
        if hasattr(event, 'guild') and event.guild:
            guild_id = event.guild.id
        elif hasattr(event, 'guild_id') and event.guild_id:
            guild_id = event.guild_id
        else:
            guild_id = None

        guild = self.guilds.get(event.guild.id) if guild_id else None
        config = guild and guild.get_config()

        # If the guild has configuration, use that (otherwise use defaults)
        if config:
            if config.commands:
                commands = list(self.bot.get_commands_for_message(
                    config.commands.mention,
                    {},
                    config.commands.prefix,
                    event.message))
        elif guild_id:
            # Otherwise, default to requiring mentions
            commands = list(self.bot.get_commands_for_message(True, {}, '', event.message))
        else:
            if ENV != 'prod':
                if not event.message.content.startswith(ENV + '!'):
                    return
                event.message.content = event.message.content[len(ENV) + 1:]

            # DM's just use the commands (no prefix/mention)
            commands = list(self.bot.get_commands_for_message(False, {}, '', event.message))

        # If we didn't find any matching commands, return
        if not len(commands):
            return

        user_level = self.get_level(event.guild, event.author)

        # Grab whether this user is a global admin
        # TODO: cache this
        global_admin = rdb.sismember('global_admins', event.author.id)

        # Iterate over commands and find a match
        for command, match in commands:
            if command.level == -1 and not global_admin:
                continue

            level = command.level

            if guild and not config and command.triggers[0] != 'setup':
                continue
            elif config and config.commands and command.plugin != self:
                if command.triggers[0] in config.commands.overrides or '*' in config.commands.overrides:
                    override = config.commands.overrides.get(
                        command.triggers[0],
                        config.commands.overrides.get('*'))
                    if override.disabled:
                        continue

                    if override.level is not None:
                        level = override.level

            if not global_admin and user_level < level:
                continue

            command.plugin.execute(CommandEvent(command, event.message, match))

            # Dispatch the command used modlog event
            if config:
                event.config = getattr(config.plugins, 'modlog', None)
                if not event.config:
                    return

                plugin = self.bot.plugins.get('ModLogPlugin')
                if plugin:
                    plugin.log_action(Actions.COMMAND_USED, event)

            return

    @Plugin.command('setup')
    def command_setup(self, event):
        """
        Setup a new Guild with Rowboat
        """
        if not event.guild:
            return event.msg.reply(':warning: this command can only be used in servers')

        # Make sure we're not already setup
        if event.guild.id in self.guilds:
            return event.msg.reply(':warning: this server is already setup')

        global_admin = rdb.sismember('global_admins', event.author.id)

        # Make sure this is the owner of the server
        if not global_admin:
            if not event.guild.owner_id == event.author.id:
                return event.msg.reply(':warning: only the server owner can setup rowboat')

        # Make sure we have admin perms
        m = event.guild.members.select_one(id=self.state.me.id)
        if not m.permissions.administrator and not global_admin:
            return event.msg.reply(':warning: bot must have the Administrator permission')

        guild = Guild.setup(event.guild)
        self.guilds[event.guild.id] = guild
        event.msg.reply(':ok_hand: successfully loaded configuration')

    @Plugin.command('about', level=CommandLevels.ADMIN)
    def command_help(self, event):
        event.msg.reply(INFO_MESSAGE)

    @Plugin.command('uptime', level=-1)
    def command_uptime(self, event):
        event.msg.reply('Rowboat was started {}'.format(
            humanize.naturaltime(datetime.utcnow() - self.startup)
        ))

    @Plugin.command('eval', level=-1)
    def command_eval(self, event):
        ctx = {
            'bot': self.bot,
            'client': self.bot.client,
            'state': self.bot.client.state,
            'event': event,
            'msg': event.msg,
            'guild': event.msg.guild,
            'channel': event.msg.channel,
            'author': event.msg.author
        }

        # Mulitline eval
        src = event.codeblock
        if src.count('\n'):
            lines = filter(bool, src.split('\n'))
            if lines[-1] and 'return' not in lines[-1]:
                lines[-1] = 'return ' + lines[-1]
            lines = '\n'.join('    ' + i for i in lines)
            code = 'def f():\n{}\nx = f()'.format(lines)
            local = {}

            try:
                exec compile(code, '<eval>', 'exec') in ctx, local
            except Exception as e:
                event.msg.reply(PY_CODE_BLOCK.format(type(e).__name__ + ': ' + str(e)))
                return

            event.msg.reply(PY_CODE_BLOCK.format(pprint.pformat(local['x'])))
        else:
            try:
                result = eval(src, ctx)
            except Exception as e:
                event.msg.reply(PY_CODE_BLOCK.format(type(e).__name__ + ': ' + str(e)))
                return

            event.msg.reply(PY_CODE_BLOCK.format(result))
<|endoftext|>


<commit_before>#Damir Jajetic, 2015, MIT licence

def predict (LD, output_dir, basename):
	import copy
	import os
	import numpy as np
	import libscores
	import data_converter
	from sklearn import preprocessing, ensemble
	from sklearn.utils import shuffle

	LD.data['X_train'], LD.data['Y_train'] = shuffle(LD.data['X_train'], LD.data['Y_train'] , random_state=1)

	Y_train = LD.data['Y_train']
	X_train = LD.data['X_train']

	Xta = np.copy(X_train)

	X_valid = LD.data['X_valid']
	X_test = LD.data['X_test']

	Xtv = np.copy(X_valid)
	Xts = np.copy(X_test)

	import xgboost as xgb
	if LD.info['name']== 'albert':
		model = xgb.XGBClassifier(max_depth=6, learning_rate=0.05, n_estimators=1800, silent=True,
				objective='binary:logistic', nthread=6, gamma=0.6,
				min_child_weight=0.7, max_delta_step=0, subsample=1,
				colsample_bytree=1, base_score=0.5, seed=0, missing=None)

	if LD.info['name']== 'dilbert':
		model = xgb.XGBClassifier(max_depth=4, learning_rate=0.1, n_estimators=1000, silent=True,
				objective='multi:softprob', nthread=-1, gamma=0,
				min_child_weight=0, max_delta_step=0, subsample=1,
				colsample_bytree=1, base_score=0.5, seed=0, missing=None)
	if LD.info['name']== 'fabert':
		model = xgb.XGBClassifier(max_depth=6, learning_rate=0.1, n_estimators=1200, silent=True,
				objective='multi:softprob', nthread=-1, gamma=0,
				min_child_weight=1, max_delta_step=0, subsample=1,
				colsample_bytree=1, base_score=0.5, seed=0, missing=None)
	if LD.info['name']== 'robert':
		model = xgb.XGBClassifier(max_depth=6, learning_rate=0.1, n_estimators=600, silent=True,
				objective='multi:softprob', nthread=-1, gamma=0,
				min_child_weight=1, max_delta_step=0, subsample=1,
				colsample_bytree=1, base_score=0.5, seed=0, missing=None)
	if LD.info['name']== 'volkert':
		from sklearn import  ensemble, preprocessing

		p = preprocessing.PolynomialFeatures()
		prep = ensemble.RandomForestRegressor(n_estimators=24, n_jobs=-1, random_state=0, verbose=1)

		prep.fit(Xta,Y_train)
		Xta = Xta [:, prep.feature_importances_.argsort()[-50:][::-1]]
		Xtv = Xtv [:, prep.feature_importances_.argsort()[-50:][::-1]]
		Xts = Xts [:, prep.feature_importances_.argsort()[-50:][::-1]]

		Xta = p.fit_transform(Xta)
		Xtv = p.fit_transform(Xtv)
		Xts = p.fit_transform(Xts)

		prep.fit(Xta,Y_train)
		Xta = Xta [:, prep.feature_importances_.argsort()[-800:][::-1]]
		Xtv = Xtv [:, prep.feature_importances_.argsort()[-800:][::-1]]
		Xts = Xts [:, prep.feature_importances_.argsort()[-800:][::-1]]

		X_train = np.hstack([X_train, Xta])
		X_valid = np.hstack([X_valid, Xtv])
		X_test = np.hstack([X_test, Xts])

		model = xgb.XGBClassifier(max_depth=6, learning_rate=0.1, n_estimators=350, silent=True,
				objective='multi:softprob', nthread=-1, gamma=0,
				min_child_weight=1, max_delta_step=0, subsample=1,
				colsample_bytree=1, base_score=0.5, seed=0, missing=None)

	model.fit(X_train, Y_train)

	preds_valid = model.predict_proba(X_valid)
	preds_test = model.predict_proba(X_test)

	import data_io
	if  LD.info['target_num']  == 1:
		preds_valid = preds_valid[:,1]
		preds_test = preds_test[:,1]

	preds_valid = np.clip(preds_valid,0,1)
	preds_test = np.clip(preds_test,0,1)

	data_io.write(os.path.join(output_dir, basename + '_valid_000.predict'), preds_valid)
	data_io.write(os.path.join(output_dir,basename + '_test_000.predict'), preds_test)

<commit_msg>Added files via upload<commit_after>#Damir Jajetic, 2016, MIT licence

def predict (LD, output_dir, basename):
	import copy
	import os
	import numpy as np
	import libscores
	import data_converter
	from sklearn import preprocessing, ensemble
	from sklearn.utils import shuffle

	LD.data['X_train'], LD.data['Y_train'] = shuffle(LD.data['X_train'], LD.data['Y_train'] , random_state=1)

	Y_train = LD.data['Y_train']
	X_train = LD.data['X_train']

	Xta = np.copy(X_train)

	X_valid = LD.data['X_valid']
	X_test = LD.data['X_test']

	Xtv = np.copy(X_valid)
	Xts = np.copy(X_test)

	import xgboost as xgb
	if LD.info['name']== 'alexis':

		model = ensemble.RandomForestClassifier(max_depth=140, n_estimators=1800, n_jobs=-1, random_state=0, verbose=0, warm_start=True)
		model2 = ensemble.RandomForestClassifier(max_depth=140, n_estimators=1800, n_jobs=-1, random_state=1, verbose=0, warm_start=True)
		model.fit(X_train, Y_train)
		model2.fit(X_train, Y_train)

		preds_valid0 = model.predict_proba(X_valid)
		preds_test0 = model.predict_proba(X_test)

		preds_valid2 = model2.predict_proba(X_valid)
		preds_test2 = model2.predict_proba(X_test)

		preds_valid0 = np.array(preds_valid0)
		preds_valid2 = np.array(preds_valid2)

		preds_test0 = np.array(preds_test0)
		preds_test2 = np.array(preds_test2)

		preds_valid = (preds_valid0 + preds_valid2)/2
		preds_test = (preds_test0 + preds_test2)/2

		preds_valid = preds_valid[:, :, 1]
		preds_valid = preds_valid.T

		preds_test = preds_test[:, :, 1]
		preds_test = preds_test.T

	if LD.info['name']== 'dionis':
		Lest = 600 #600 will consume cca 250 GB of RAM, use 50 for similar result
		Lest = 50

		model = ensemble.RandomForestClassifier( n_jobs=-1, n_estimators=Lest, random_state=0)
		model.fit(X_train, Y_train)
		preds_valid0 = model.predict_proba(X_valid)
		preds_test0 = model.predict_proba(X_test)

		model = ensemble.RandomForestClassifier( n_jobs=-1, n_estimators=Lest, random_state=1)
		model.fit(X_train, Y_train)
		preds_valid1 = model.predict_proba(X_valid)
		preds_test1 = model.predict_proba(X_test)

		model = ensemble.RandomForestClassifier( n_jobs=-1, n_estimators=Lest, random_state=2)
		model.fit(X_train, Y_train)
		preds_valid2 = model.predict_proba(X_valid)
		preds_test2 = model.predict_proba(X_test)

		model = ensemble.RandomForestClassifier( n_jobs=-1, n_estimators=Lest, random_state=3)
		model.fit(X_train, Y_train)
		preds_valid3 = model.predict_proba(X_valid)
		preds_test3 = model.predict_proba(X_test)

		model = ensemble.RandomForestClassifier( n_jobs=-1, n_estimators=Lest, random_state=4)
		model.fit(X_train, Y_train)
		preds_valid4 = model.predict_proba(X_valid)
		preds_test4 = model.predict_proba(X_test)

		preds_valid = (preds_valid0 + preds_valid1 + preds_valid2 + preds_valid3 + preds_valid4) # /5 should be included (bug)
		preds_test = (preds_test0 + preds_test1 + preds_test2 + preds_test3 + preds_test4) # /5 should be included (bug)

	if LD.info['name']== 'grigoris':
		model = ensemble.RandomForestClassifier(criterion='entropy', max_features=0.05, max_depth=5, n_estimators=120, n_jobs=-1, random_state=0, verbose=0)
		model2 = linear_model.LogisticRegression(penalty='l1', random_state=1, n_jobs=-1, C=0.008)
		model3 = ensemble.RandomForestClassifier(criterion='entropy', max_features=0.05, max_depth=5, n_estimators=120, n_jobs=-1, random_state=1, verbose=0)
		model4 = ensemble.RandomForestClassifier(criterion='entropy', max_features=0.05, max_depth=5, n_estimators=120, n_jobs=-1, random_state=2, verbose=0)

		preds_valid = np.zeros((X_valid.shape[0], Y_train.shape[1]))
		preds_test = np.zeros((X_test.shape[0], Y_train.shape[1]))
		for pyt in range(Y_train.shape[1]):
			print pyt
			ytp = Y_train[:, pyt]
			model.fit(X_train, ytp)
			model2.fit(X_train, ytp)
			model3.fit(X_train, ytp)
			model4.fit(X_train, ytp)

			preds1v= model.predict_proba (X_valid)[:, 1]
			preds2v= model2.predict_proba (X_valid)[:, 1]
			preds3v= model3.predict_proba (X_valid)[:, 1]
			preds4v= model4.predict_proba (X_valid)[:, 1]
			predsv = (preds1v + preds2v + preds3v + preds4v)/4
			preds_valid[:, pyt] = predsv

			preds1t= model.predict_proba (X_test)[:, 1]
			preds2t= model2.predict_proba (X_test)[:, 1]
			preds3t= model3.predict_proba (X_test)[:, 1]
			preds4t= model4.predict_proba (X_test)[:, 1]
			predst = (preds1t + preds2t + preds3t + preds4t)/4
			preds_test[:, pyt] = predst

	if LD.info['name']== 'jannis':
		Xd = X_train[Y_train==0]
		yd = Y_train[Y_train==0]

		for a in range(18):
			X_train = np.vstack([X_train, Xd])
			Y_train = np.hstack([Y_train, yd])

		Xd = X_train[Y_train==2]
		yd = Y_train[Y_train==2]

		X_train = np.vstack([X_train, Xd])
		Y_train = np.hstack([Y_train, yd])

		Y_train_raw = np.array(data_converter.convert_to_bin(Y_train, len(np.unique(Y_train)), False))

		preds_valid = np.zeros((X_valid.shape[0], Y_train_raw.shape[1]))
		preds_test = np.zeros((X_test.shape[0], Y_train_raw.shape[1]))
		for pyt in range(Y_train_raw.shape[1]):
			if pyt == 0:
				Lbs = 0.2
			else:
				Lbs = 0.5

			model = xgb.XGBClassifier(max_depth=30, learning_rate=0.05, n_estimators=100, silent=True,
				objective='binary:logistic', nthread=-1, gamma=0,
				min_child_weight=80, max_delta_step=1, subsample=1,
				colsample_bytree=1, base_score=Lbs, seed=0, missing=None)

			ytp = Y_train_raw[:, pyt]
			model.fit(X_train, ytp)

			preds1v= model.predict_proba (X_valid)[:, 1]
			preds_valid[:, pyt] = preds1v

			preds1t= model.predict_proba (X_test)[:, 1]
			preds_test[:, pyt] = preds1t

	if LD.info['name']== 'wallis':
		model = naive_bayes.MultinomialNB(alpha=0.02)

		model2 = xgb.XGBClassifier(max_depth=5, learning_rate=0.05, n_estimators=1200, silent=True,
				objective='multi:softprob', nthread=-1, gamma=0,
				min_child_weight=1, max_delta_step=0, subsample=1,
				colsample_bytree=1, base_score=0.5, seed=0, missing=None)

		model.fit(X_train, Y_train)
		preds_valid1 = model.predict_proba(X_valid)
		preds_test1 = model.predict_proba(X_test)

		model2.fit(X_train, Y_train)
		preds_valid2 = model2.predict_proba(X_valid)
		preds_test2 = model2.predict_proba(X_test)

		preds_valid = (preds_valid1 +preds_valid2)/2
		preds_test = (preds_test1 +preds_test2)/2

	import data_io
	if  LD.info['target_num']  == 1:
		preds_valid = preds_valid[:,1]
		preds_test = preds_test[:,1]

	preds_valid = np.clip(preds_valid,0,1)
	preds_test = np.clip(preds_test,0,1)

	data_io.write(os.path.join(output_dir, basename + '_valid_000.predict'), preds_valid)
	data_io.write(os.path.join(output_dir,basename + '_test_000.predict'), preds_test)

<|endoftext|>