<commit_before>import copy
import logging
import os
import re

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from ConfigParser import Error as ConfigParserError, SafeConfigParser
from decimal import Decimal

from django import forms
from django.conf import settings as django_settings
from django.db.models.loading import get_model
from django.utils import importlib
from django.utils.datastructures import SortedDict

__all__ = ('AVAILABLE_SETTINGS', 'auth_permitted', 'parse_config')

DEFAULT_SETTINGS_FILENAME = 'settings.cfg'
logger = logging.getLogger('setman')

class ConfigParser(SafeConfigParser, object):
    """
    Customize default behavior for config parser instances to support config
    files without sections at all.
    """
    no_sections_mode = False
    optionxform = lambda _, value: value

    def _read(self, fp, fpname):
        """
        If "No Sections Mode" enabled - add global section as first line of
        file handler.
        """
        if self.no_sections_mode:
            global_section = StringIO()
            global_section.write('[DEFAULT]\n')
            global_section.write(fp.read())
            global_section.seek(0)
            fp = global_section

        return super(ConfigParser, self)._read(fp, fpname)

class Setting(object):
    """
    Base class for setting values that can provided in configuration definition
    file.

    The class has next attributes:

    * ``name``
    * ``type``
    * ``default``
    * ``required``
    * ``label``
    * ``help_text``
    * ``validators``
    * ``field``
    * ``field_args``
    * ``field_kwargs``

    The last three attributes can be provided only in Python module, when all
    other attrs can read from configuration definition file.
    """
    default = None
    field_args = ('label', 'help_text', 'initial', 'required', 'validators')
    field_klass = None
    field_kwargs = {}
    help_text = None
    label = None
    name = None
    required = True
    type = None
    validators = None

    def __init__(self, **kwargs):
        self._validators = kwargs.pop('validators', None)
        restricted = ('field_klass', 'field_args', 'field_kwargs',
                      'validators')

        for key, _ in kwargs.items():
            if not hasattr(self, key):
                kwargs.pop(key)

            if key in restricted:
                kwargs.pop(key)

        self.__dict__.update(kwargs)
        self.required = force_bool(self.required)

    def __repr__(self):
        return u'<%s: %s>' % (self.__class__.__name__, self.__unicode__())

    def __unicode__(self):
        return u'%s = %r' % (self.name, self.initial)

    @property
    def initial(self):
        """
        Read real setting value from database or if impossible - just send
        default setting value.
        """
        from setman import settings
        return getattr(settings, self.name, self.default)

    def to_field(self, **kwargs):
        """
        Convert current setting instance to form field.

        You should provide ``kwargs`` and all values from here would be used
        when initing ``field`` instance instead of ``Setting`` attributes.
        """
        if not self.field_klass:
            raise ValueError('Please, supply `field_klass` attribute first.')

        field_kwargs = {}

        for arg in self.field_args:
            value = kwargs[arg] if arg in kwargs else getattr(self, arg)
            field_kwargs.update({arg: value})

        field_kwargs.update(**self.field_kwargs)
        return self.field_klass(**field_kwargs)

    def to_python(self, value):
        """
        Convert setting value to necessary Python type. By default, returns
        same value without any conversion.
        """
        return value

    @property
    def validators(self):
        """
        Lazy loaded validators.
        """
        cache_key = '_validators_cache'
        if not hasattr(self, cache_key):
            setattr(self, cache_key, self._parse_validators(self._validators))
        return getattr(self, cache_key)

    def _parse_validators(self, value):
        """
        Parse validators string and try to convert it to list with actual
        validator functions.
        """
        if not value:
            return []

        items = map(lambda item: item.strip(), value.split(','))
        validators = []

        for item in items:
            try:
                validator = load_from_path(item)
            except (AttributeError, ImportError):
                logger.exception('Cannot load %r validator for %s setting.',
                                 item, self.name)
                continue

            validators.append(validator)

        return validators

class SettingTypeDoesNotExist(Exception):
    """
    Simple exception that raised when user tried to load not supported setting
    type from configuration definition file.
    """

class BooleanSetting(Setting):
    """
    Boolean setting.
    """
    field_klass = forms.BooleanField
    required = False
    type = 'boolean'

    def __init__(self, **kwargs):
        super(BooleanSetting, self).__init__(**kwargs)
        self.default = self.to_python(self.default)

    def to_python(self, value):
        """
        Convert string to the boolean type.
        """
        return force_bool(value)

class ChoiceSetting(Setting):
    """
    Choice setting.
    """
    choices = None
    field_args = Setting.field_args + ('choices', )
    field_klass = forms.ChoiceField
    type = 'choice'

    def __init__(self, **kwargs):
        self._choices = kwargs.pop('choices', None)
        super(ChoiceSetting, self).__init__(**kwargs)

    @property
    def choices(self):
        """
        Lazy loaded choices.
        """
        cache_key = '_choices_cache'
        if not hasattr(self, cache_key):
            setattr(self, cache_key, self._parse_choices(self._choices))
        return getattr(self, cache_key)

    def _parse_choices(self, value):
        """
        Convert string value to valid choices tuple.

        **Supported formats:**

        * a, b, c
        * (a, A), (b, B), (c, C)
        * a { b, c }, d { e, f }
        * A { (b, B), (c, C) }, D { (e, E), (f, F) }
        * path.to.CHOICES
        * path.to.Model.CHOICES
        * app.Model.CHOICES

        """
        # Start parsing with internal choices
        if not ',' in value and '.' in value:
            # Choices tuple should be last part of value
            path, attr = value.rsplit('.', 1)

            # Try to process path as ``app.Model`` definition
            model = None

            try:
                app, model = path.split('.')
            except ValueError:
                pass
            else:
                model = get_model(app, model)

            # If cannot process path as ``app.Model`` just load it as module
            # or as class from module
            if model is None:
                try:
                    module = importlib.import_module(path)
                except ImportError:
                    try:
                        module = load_from_path(path)
                    except (AttributeError, ImportError):
                        logger.exception('Cannot load choices from %r path',
                                         value)
                        return ()
            else:
                module = model

            # And finally, try to get choices attr in module or model
            try:
                choices = getattr(module, attr)
            except AttributeError:
                logger.exception('Cannot load choices from %r path', value)
                return ()
        elif not '{' in value and not '}' in value:
            # Parse choice with labels
            label_re = re.compile(r'\(([^,]+),\s+([^\)]+)\)', re.M)
            found = label_re.findall(value)

            if found:
                choices = found
            # If nothing found by regex, just split value by comma and
            # duplicate resulted items
            else:
                choices = map(lambda item: (item.strip(), item.strip()),
                              value.split(','))
        else:
            # Parse groups
            groups_re = re.compile(r'([^{]+){([^}]+)},?', re.M)
            found = groups_re.findall(value)

            if found:
                choices = []

                for group, data in found:
                    group = group.strip()
                    choices.append((group, self._parse_choices(data.strip())))
            else:
                logger.error('Cannot parse choices from %r', value)
                return ()

        return tuple(choices)

class DecimalSetting(Setting):
    """
    Decimal setting.
    """
    decimal_places = None
    field_args = Setting.field_args + ('decimal_places', 'max_digits',
                                       'max_value', 'min_value')
    field_klass = forms.DecimalField
    max_digits = None
    max_value = None
    min_value = None
    type = 'decimal'

    def __init__(self, **kwargs):
        super(DecimalSetting, self).__init__(**kwargs)

        int_setting = IntSetting()
        self.decimal_places = int_setting.to_python(self.decimal_places)
        self.max_digits = int_setting.to_python(self.max_digits)

        self.default = self.to_python(self.default)
        self.max_value = self.to_python(self.max_value)
        self.min_value = self.to_python(self.min_value)

    def to_python(self, value):
        if value is None:
            return value
        return Decimal(value)

class IntSetting(Setting):
    """
    Integer setting.
    """
    field_args = Setting.field_args + ('max_value', 'min_value')
    field_klass = forms.IntegerField
    max_value = None
    min_value = None
    type = 'int'

    def __init__(self, **kwargs):
        super(IntSetting, self).__init__(**kwargs)
        self.default = self.to_python(self.default)
        self.max_value = self.to_python(self.max_value)
        self.min_value = self.to_python(self.min_value)

    def to_python(self, value):
        try:
            return int(value)
        except (TypeError, ValueError):
            return None

class FloatSetting(IntSetting):
    """
    Float setting.
    """
    field_klass = forms.FloatField
    type = 'float'

    def to_python(self, value):
        try:
            return float(value)
        except (TypeError, ValueError):
            return None

class StringSetting(Setting):
    """
    String setting.
    """
    field_args = Setting.field_args + ('max_length', 'min_length')
    field_klass = forms.CharField
    max_length = None
    min_length = None
    regex = None
    type = 'string'

    def to_field(self, **kwargs):
        """
        Use ``RegexField`` for string setting if ``regex`` was filled in
        configuration definition file.
        """
        if self.regex:
            if not 'regex' in self.field_args:
                self.field_args = self.field_args + ('regex', )
            self.field_klass = forms.RegexField
        return super(StringSetting, self).to_field(**kwargs)

class SettingsContainer(object):

    def __init__(self, path):
        self._data = []
        self.path = path

    def __iter__(self):
        return (item for item in self._data)

    def __len__(self):
        return len(self._data)

    def append(self, value):
        self._data.append(value)
        setattr(self, value.name, value)

def auth_permitted(user):
    """
    Check that the user can have access to the view.
    """
    default = lambda user: user.is_superuser
    func = getattr(django_settings, 'SETMAN_AUTH_PERMITTED', default)
    return func(user)

def data_to_setting(data, additional_types=None):
    """
    Convert data dict to setting instance.
    """
    additional_types = additional_types or []
    setting = None
    setting_type = data.get('type')

    all_values = globals().values() + additional_types

    for value in all_values:
        try:
            if not issubclass(value, Setting):
                continue
        except TypeError:
            continue

        if not value.type or not setting_type or \
           value.type.lower() != setting_type.lower():
            continue

        setting = value(**data)

    if setting is None:
        raise SettingTypeDoesNotExist('%r setting type not found.' % \
                                      setting_type)

    return setting

def force_bool(value):
    """
    Convert string value to boolean instance.
    """
    if isinstance(value, (bool, int)):
        return bool(value)

    boolean_states = ConfigParser._boolean_states
    if not value.lower() in boolean_states:
        return None

    return boolean_states[value.lower()]

def load_from_path(path):
    """
    Load class or function from string path.
    """
    module, attr = path.rsplit('.', 1)
    mod = importlib.import_module(module)
    return getattr(mod, attr)

def parse_config(path=None):
    """
    Parse Configuration Definition File.

    In most cases this file needs to be placed in same folder where project
    settings module exist and named as ``settings.cfg``.

    But you can customize things with using ``SETMAN_SETTINGS_FILE`` option.
    Provide there path where settings file actually placed.

    Also current function can called with ``path`` string.
    """
    additional_types = getattr(django_settings, 'SETMAN_ADDITIONAL_TYPES', ())
    additional_setting_types = []
    default_values = {}

    for item in additional_types:
        try:
            additional_type = load_from_path(item)
        except (AttributeError, TypeError):
            logger.exception('Cannot load %r additional setting type from ' \
                             'configuration.', item)

        additional_setting_types.append(additional_type)

    if path is None:
        path = getattr(django_settings, 'SETMAN_SETTINGS_FILE', None)

        if path is None:
            module = importlib.import_module(django_settings.SETTINGS_MODULE)
            dirname = os.path.dirname(os.path.normpath(module.__file__))
            path = os.path.join(dirname, DEFAULT_SETTINGS_FILENAME)

    empty_settings = SettingsContainer(path)

    if not os.path.isfile(path):
        logger.error('Cannot read configuration definition file at %r. Exit ' \
                     'from parsing!', path)
        return empty_settings

    # Use ``SortedDict`` instance for reading sections on config file instead
    # of default ``dict`` that can shuffle the sections.
    config = ConfigParser(dict_type=SortedDict)

    # First we need to read default values file
    default_values_file = \
        getattr(django_settings, 'SETMAN_DEFAULT_VALUES_FILE', None)

    if default_values_file:
        config.no_sections_mode = True

        try:
            config.read(default_values_file)
        except ConfigParserError:
            logger.exception('Cannot read default values from %r',
                             default_values_file)
        else:
            default_values = config.defaults()
        finally:
            config.no_sections_mode = False

    # Only then really read from config definition file
    try:
        config.read(path)
    except ConfigParserError:
        logger.exception('Cannot parse configuration definition file from ' \
                         '%r', path)
        return empty_settings

    settings = copy.deepcopy(empty_settings)

    for setting in config.sections():
        data = dict(config.items(setting))
        data.update({'name': setting})

        if setting in default_values:
            data.update({'default': default_values[setting]})

        try:
            setting = data_to_setting(data, additional_setting_types)
        except SettingTypeDoesNotExist:
            logger.exception('Cannot find proper setting class for %r type',
                             data.get('type'))
            return empty_settings

        settings.append(setting)

    return settings

AVAILABLE_SETTINGS = parse_config()
<commit_msg>Fixed a bug in Decimal field which did not load the exact decimal value from settings.cfg but the rounded value<commit_after>import copy
import logging
import os
import re

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from ConfigParser import Error as ConfigParserError, SafeConfigParser
from decimal import Decimal

from django import forms
from django.conf import settings as django_settings
from django.db.models.loading import get_model
from django.utils import importlib
from django.utils.datastructures import SortedDict

__all__ = ('AVAILABLE_SETTINGS', 'auth_permitted', 'parse_config')

DEFAULT_SETTINGS_FILENAME = 'settings.cfg'
logger = logging.getLogger('setman')

class ConfigParser(SafeConfigParser, object):
    """
    Customize default behavior for config parser instances to support config
    files without sections at all.
    """
    no_sections_mode = False
    optionxform = lambda _, value: value

    def _read(self, fp, fpname):
        """
        If "No Sections Mode" enabled - add global section as first line of
        file handler.
        """
        if self.no_sections_mode:
            global_section = StringIO()
            global_section.write('[DEFAULT]\n')
            global_section.write(fp.read())
            global_section.seek(0)
            fp = global_section

        return super(ConfigParser, self)._read(fp, fpname)

class Setting(object):
    """
    Base class for setting values that can provided in configuration definition
    file.

    The class has next attributes:

    * ``name``
    * ``type``
    * ``default``
    * ``required``
    * ``label``
    * ``help_text``
    * ``validators``
    * ``field``
    * ``field_args``
    * ``field_kwargs``

    The last three attributes can be provided only in Python module, when all
    other attrs can read from configuration definition file.
    """
    default = None
    field_args = ('label', 'help_text', 'initial', 'required', 'validators')
    field_klass = None
    field_kwargs = {}
    help_text = None
    label = None
    name = None
    required = True
    type = None
    validators = None

    def __init__(self, **kwargs):
        self._validators = kwargs.pop('validators', None)
        restricted = ('field_klass', 'field_args', 'field_kwargs',
                      'validators')

        for key, _ in kwargs.items():
            if not hasattr(self, key):
                kwargs.pop(key)

            if key in restricted:
                kwargs.pop(key)

        self.__dict__.update(kwargs)
        self.required = force_bool(self.required)

    def __repr__(self):
        return u'<%s: %s>' % (self.__class__.__name__, self.__unicode__())

    def __unicode__(self):
        return u'%s = %r' % (self.name, self.initial)

    @property
    def initial(self):
        """
        Read real setting value from database or if impossible - just send
        default setting value.
        """
        from setman import settings
        return getattr(settings, self.name, self.default)

    def to_field(self, **kwargs):
        """
        Convert current setting instance to form field.

        You should provide ``kwargs`` and all values from here would be used
        when initing ``field`` instance instead of ``Setting`` attributes.
        """
        if not self.field_klass:
            raise ValueError('Please, supply `field_klass` attribute first.')

        field_kwargs = {}

        for arg in self.field_args:
            value = kwargs[arg] if arg in kwargs else getattr(self, arg)
            field_kwargs.update({arg: value})

        field_kwargs.update(**self.field_kwargs)
        return self.field_klass(**field_kwargs)

    def to_python(self, value):
        """
        Convert setting value to necessary Python type. By default, returns
        same value without any conversion.
        """
        return value

    @property
    def validators(self):
        """
        Lazy loaded validators.
        """
        cache_key = '_validators_cache'
        if not hasattr(self, cache_key):
            setattr(self, cache_key, self._parse_validators(self._validators))
        return getattr(self, cache_key)

    def _parse_validators(self, value):
        """
        Parse validators string and try to convert it to list with actual
        validator functions.
        """
        if not value:
            return []

        items = map(lambda item: item.strip(), value.split(','))
        validators = []

        for item in items:
            try:
                validator = load_from_path(item)
            except (AttributeError, ImportError):
                logger.exception('Cannot load %r validator for %s setting.',
                                 item, self.name)
                continue

            validators.append(validator)

        return validators

class SettingTypeDoesNotExist(Exception):
    """
    Simple exception that raised when user tried to load not supported setting
    type from configuration definition file.
    """

class BooleanSetting(Setting):
    """
    Boolean setting.
    """
    field_klass = forms.BooleanField
    required = False
    type = 'boolean'

    def __init__(self, **kwargs):
        super(BooleanSetting, self).__init__(**kwargs)
        self.default = self.to_python(self.default)

    def to_python(self, value):
        """
        Convert string to the boolean type.
        """
        return force_bool(value)

class ChoiceSetting(Setting):
    """
    Choice setting.
    """
    choices = None
    field_args = Setting.field_args + ('choices', )
    field_klass = forms.ChoiceField
    type = 'choice'

    def __init__(self, **kwargs):
        self._choices = kwargs.pop('choices', None)
        super(ChoiceSetting, self).__init__(**kwargs)

    @property
    def choices(self):
        """
        Lazy loaded choices.
        """
        cache_key = '_choices_cache'
        if not hasattr(self, cache_key):
            setattr(self, cache_key, self._parse_choices(self._choices))
        return getattr(self, cache_key)

    def _parse_choices(self, value):
        """
        Convert string value to valid choices tuple.

        **Supported formats:**

        * a, b, c
        * (a, A), (b, B), (c, C)
        * a { b, c }, d { e, f }
        * A { (b, B), (c, C) }, D { (e, E), (f, F) }
        * path.to.CHOICES
        * path.to.Model.CHOICES
        * app.Model.CHOICES

        """
        # Start parsing with internal choices
        if not ',' in value and '.' in value:
            # Choices tuple should be last part of value
            path, attr = value.rsplit('.', 1)

            # Try to process path as ``app.Model`` definition
            model = None

            try:
                app, model = path.split('.')
            except ValueError:
                pass
            else:
                model = get_model(app, model)

            # If cannot process path as ``app.Model`` just load it as module
            # or as class from module
            if model is None:
                try:
                    module = importlib.import_module(path)
                except ImportError:
                    try:
                        module = load_from_path(path)
                    except (AttributeError, ImportError):
                        logger.exception('Cannot load choices from %r path',
                                         value)
                        return ()
            else:
                module = model

            # And finally, try to get choices attr in module or model
            try:
                choices = getattr(module, attr)
            except AttributeError:
                logger.exception('Cannot load choices from %r path', value)
                return ()
        elif not '{' in value and not '}' in value:
            # Parse choice with labels
            label_re = re.compile(r'\(([^,]+),\s+([^\)]+)\)', re.M)
            found = label_re.findall(value)

            if found:
                choices = found
            # If nothing found by regex, just split value by comma and
            # duplicate resulted items
            else:
                choices = map(lambda item: (item.strip(), item.strip()),
                              value.split(','))
        else:
            # Parse groups
            groups_re = re.compile(r'([^{]+){([^}]+)},?', re.M)
            found = groups_re.findall(value)

            if found:
                choices = []

                for group, data in found:
                    group = group.strip()
                    choices.append((group, self._parse_choices(data.strip())))
            else:
                logger.error('Cannot parse choices from %r', value)
                return ()

        return tuple(choices)

class DecimalSetting(Setting):
    """
    Decimal setting.
    """
    decimal_places = None
    field_args = Setting.field_args + ('decimal_places', 'max_digits',
                                       'max_value', 'min_value')
    field_klass = forms.DecimalField
    max_digits = None
    max_value = None
    min_value = None
    type = 'decimal'

    def __init__(self, **kwargs):
        super(DecimalSetting, self).__init__(**kwargs)

        int_setting = IntSetting()
        self.decimal_places = int_setting.to_python(self.decimal_places)
        self.max_digits = int_setting.to_python(self.max_digits)

        self.default = self.to_python(self.default)
        self.max_value = self.to_python(self.max_value)
        self.min_value = self.to_python(self.min_value)

    def to_python(self, value):
        if value is None:
            return value
        return Decimal(str(value))

class IntSetting(Setting):
    """
    Integer setting.
    """
    field_args = Setting.field_args + ('max_value', 'min_value')
    field_klass = forms.IntegerField
    max_value = None
    min_value = None
    type = 'int'

    def __init__(self, **kwargs):
        super(IntSetting, self).__init__(**kwargs)
        self.default = self.to_python(self.default)
        self.max_value = self.to_python(self.max_value)
        self.min_value = self.to_python(self.min_value)

    def to_python(self, value):
        try:
            return int(value)
        except (TypeError, ValueError):
            return None

class FloatSetting(IntSetting):
    """
    Float setting.
    """
    field_klass = forms.FloatField
    type = 'float'

    def to_python(self, value):
        try:
            return float(value)
        except (TypeError, ValueError):
            return None

class StringSetting(Setting):
    """
    String setting.
    """
    field_args = Setting.field_args + ('max_length', 'min_length')
    field_klass = forms.CharField
    max_length = None
    min_length = None
    regex = None
    type = 'string'

    def to_field(self, **kwargs):
        """
        Use ``RegexField`` for string setting if ``regex`` was filled in
        configuration definition file.
        """
        if self.regex:
            if not 'regex' in self.field_args:
                self.field_args = self.field_args + ('regex', )
            self.field_klass = forms.RegexField
        return super(StringSetting, self).to_field(**kwargs)

class SettingsContainer(object):

    def __init__(self, path):
        self._data = []
        self.path = path

    def __iter__(self):
        return (item for item in self._data)

    def __len__(self):
        return len(self._data)

    def append(self, value):
        self._data.append(value)
        setattr(self, value.name, value)

def auth_permitted(user):
    """
    Check that the user can have access to the view.
    """
    default = lambda user: user.is_superuser
    func = getattr(django_settings, 'SETMAN_AUTH_PERMITTED', default)
    return func(user)

def data_to_setting(data, additional_types=None):
    """
    Convert data dict to setting instance.
    """
    additional_types = additional_types or []
    setting = None
    setting_type = data.get('type')

    all_values = globals().values() + additional_types

    for value in all_values:
        try:
            if not issubclass(value, Setting):
                continue
        except TypeError:
            continue

        if not value.type or not setting_type or \
           value.type.lower() != setting_type.lower():
            continue

        setting = value(**data)

    if setting is None:
        raise SettingTypeDoesNotExist('%r setting type not found.' % \
                                      setting_type)

    return setting

def force_bool(value):
    """
    Convert string value to boolean instance.
    """
    if isinstance(value, (bool, int)):
        return bool(value)

    boolean_states = ConfigParser._boolean_states
    if not value.lower() in boolean_states:
        return None

    return boolean_states[value.lower()]

def load_from_path(path):
    """
    Load class or function from string path.
    """
    module, attr = path.rsplit('.', 1)
    mod = importlib.import_module(module)
    return getattr(mod, attr)

def parse_config(path=None):
    """
    Parse Configuration Definition File.

    In most cases this file needs to be placed in same folder where project
    settings module exist and named as ``settings.cfg``.

    But you can customize things with using ``SETMAN_SETTINGS_FILE`` option.
    Provide there path where settings file actually placed.

    Also current function can called with ``path`` string.
    """
    additional_types = getattr(django_settings, 'SETMAN_ADDITIONAL_TYPES', ())
    additional_setting_types = []
    default_values = {}

    for item in additional_types:
        try:
            additional_type = load_from_path(item)
        except (AttributeError, TypeError):
            logger.exception('Cannot load %r additional setting type from ' \
                             'configuration.', item)

        additional_setting_types.append(additional_type)

    if path is None:
        path = getattr(django_settings, 'SETMAN_SETTINGS_FILE', None)

        if path is None:
            module = importlib.import_module(django_settings.SETTINGS_MODULE)
            dirname = os.path.dirname(os.path.normpath(module.__file__))
            path = os.path.join(dirname, DEFAULT_SETTINGS_FILENAME)

    empty_settings = SettingsContainer(path)

    if not os.path.isfile(path):
        logger.error('Cannot read configuration definition file at %r. Exit ' \
                     'from parsing!', path)
        return empty_settings

    # Use ``SortedDict`` instance for reading sections on config file instead
    # of default ``dict`` that can shuffle the sections.
    config = ConfigParser(dict_type=SortedDict)

    # First we need to read default values file
    default_values_file = \
        getattr(django_settings, 'SETMAN_DEFAULT_VALUES_FILE', None)

    if default_values_file:
        config.no_sections_mode = True

        try:
            config.read(default_values_file)
        except ConfigParserError:
            logger.exception('Cannot read default values from %r',
                             default_values_file)
        else:
            default_values = config.defaults()
        finally:
            config.no_sections_mode = False

    # Only then really read from config definition file
    try:
        config.read(path)
    except ConfigParserError:
        logger.exception('Cannot parse configuration definition file from ' \
                         '%r', path)
        return empty_settings

    settings = copy.deepcopy(empty_settings)

    for setting in config.sections():
        data = dict(config.items(setting))
        data.update({'name': setting})

        if setting in default_values:
            data.update({'default': default_values[setting]})

        try:
            setting = data_to_setting(data, additional_setting_types)
        except SettingTypeDoesNotExist:
            logger.exception('Cannot find proper setting class for %r type',
                             data.get('type'))
            return empty_settings

        settings.append(setting)

    return settings

AVAILABLE_SETTINGS = parse_config()
<|endoftext|>


<commit_before>import os.path as path
import re

import Levenshtein

from .sanitize import sanitize
from .tmdb import search

def _gather(filename, directory=None, titles={}):
    # Sanitize the input filename
    name, year = sanitize(filename)

    # Start with a basic search
    results = search(name, year)

    if year is not None and len(results) == 0:
        # If no results are found when year is present,
        # allow a tolerance of 1 in the year
        results = search(name, year + 1)
        results = results + search(name, year - 1)

    # Try to find a result with zero error and return
    zero_distance_results = []
    for i, result in enumerate(results):
        distance = Levenshtein.distance(
            unicode(re.sub('[^a-zA-Z0-9]', '', name.lower())),
            unicode(re.sub('[^a-zA-Z0-9]', '', result['title'].lower()))
        )

        # Update the results with the distance
        result['distance'] = distance
        results[i]['distance'] = distance

        # Update the results with year
        result['with_year'] = (year is not None)
        results[i]['with_year'] = (year is not None)

        # Add count field to the result
        result['count'] = 1
        results[i]['count'] = 1

        if distance == 0:
            zero_distance_results.append(result)

    if len(zero_distance_results) > 0:
        # Directly return results with zero error
        return zero_distance_results

    if year is not None and len(results) > 0:
        # Directly return results which were queried with year
        return results

    # If neither zero distance results are present nor is the year,
    # accumulate results from directory one level up
    if directory is not None:
        dirname = directory.split('/')[-1]
        results_from_directory = _gather(dirname)

        results_to_be_removed = []

        # Increment count for all duplicate results
        for i, r1 in enumerate(results):
            for r2 in results_from_directory:
                if r1['popularity'] == r2['popularity']:
                    # Check with popularity since title can be duplicate
                    results[i]['count'] += r2['count']
                    results_from_directory.remove(r2)
                    break

        results = results + results_from_directory

    return results

def identify(filename, directory=None):
    if directory == '' or directory == '.' or directory == '..':
        directory = None

    results = _gather(filename, directory)
    for i, result in enumerate(results):
        # Add year to all the results
        try:
            results[i]['year'] = re.findall(
                '[0-9]{4}', result['release_date'])[0]
        except TypeError:
            results[i]['year'] = None

    max_distance = 1 + max([result['distance'] for result in results])
    return sorted(
        results,
        key=lambda r: ((r['count'] ** 1.1) *
                       ((max_distance - r['distance'])) *
                       ((1 + r['with_year'])) *
                       ((r['popularity']))),
        reverse=True
    )
<commit_msg>Fix error when empty array is used to call max()<commit_after>import os.path as path
import re

import Levenshtein

from .sanitize import sanitize
from .tmdb import search

def _gather(filename, directory=None, titles={}):
    # Sanitize the input filename
    name, year = sanitize(filename)

    # Start with a basic search
    results = search(name, year)

    if year is not None and len(results) == 0:
        # If no results are found when year is present,
        # allow a tolerance of 1 in the year
        results = search(name, year + 1)
        results = results + search(name, year - 1)

    # Try to find a result with zero error and return
    zero_distance_results = []
    for i, result in enumerate(results):
        distance = Levenshtein.distance(
            unicode(re.sub('[^a-zA-Z0-9]', '', name.lower())),
            unicode(re.sub('[^a-zA-Z0-9]', '', result['title'].lower()))
        )

        # Update the results with the distance
        result['distance'] = distance
        results[i]['distance'] = distance

        # Update the results with year
        result['with_year'] = (year is not None)
        results[i]['with_year'] = (year is not None)

        # Add count field to the result
        result['count'] = 1
        results[i]['count'] = 1

        if distance == 0:
            zero_distance_results.append(result)

    if len(zero_distance_results) > 0:
        # Directly return results with zero error
        return zero_distance_results

    if year is not None and len(results) > 0:
        # Directly return results which were queried with year
        return results

    # If neither zero distance results are present nor is the year,
    # accumulate results from directory one level up
    if directory is not None:
        dirname = directory.split('/')[-1]
        results_from_directory = _gather(dirname)

        results_to_be_removed = []

        # Increment count for all duplicate results
        for i, r1 in enumerate(results):
            for r2 in results_from_directory:
                if r1['popularity'] == r2['popularity']:
                    # Check with popularity since title can be duplicate
                    results[i]['count'] += r2['count']
                    results_from_directory.remove(r2)
                    break

        results = results + results_from_directory

    return results

def identify(filename, directory=None):
    if directory == '' or directory == '.' or directory == '..':
        directory = None

    results = _gather(filename, directory)
    for i, result in enumerate(results):
        # Add year to all the results
        try:
            results[i]['year'] = re.findall(
                '[0-9]{4}', result['release_date'])[0]
        except TypeError:
            results[i]['year'] = None

    if len(results) == 0:
        return []

    max_distance = 1 + max([result['distance'] for result in results])
    return sorted(
        results,
        key=lambda r: ((r['count'] ** 1.1) *
                       ((max_distance - r['distance'])) *
                       ((1 + r['with_year'])) *
                       ((r['popularity']))),
        reverse=True
    )
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

#:Current version.
__version__ = '1.0.dev0'

from .polyhedron import *
from .lie_group import *
from .ressayre import *
from .combinat import *

from . import qmp
from . import third_party
<commit_msg>Version 1.0 released.<commit_after>from __future__ import absolute_import, print_function

#:Current version.
__version__ = '1.0'

from .polyhedron import *
from .lie_group import *
from .ressayre import *
from .combinat import *

from . import qmp
from . import third_party
<|endoftext|>


<commit_before>from unittest import mock

import pytest
from celery.exceptions import Ignore

from .job import Job, DEBUG, INFO, WARN, ERROR

def test_logging():
    """On each log entry update_state is called with the log."""
    job = Job()
    job.begin()

    current = {}

    def update_state(self, state, meta):
        current["state"] = state
        current["meta"] = meta

    with mock.patch(
        "celery.Task.update_state", new=update_state,
    ):

        for index, level in enumerate(["error", "warn", "info", "debug"]):
            getattr(job, level)("{} message".format(level))

            log = job.log_entries[-1]
            assert "time" in log
            assert log["level"] == index
            assert log["message"] == "{} message".format(level)
            assert current["state"] == "RUNNING"
            assert current["meta"]["log"] == job.log_entries

def test_success():
    """Returns both result and log."""
    job = Job()
    job.begin()
    returns = job.success(42)
    assert returns["result"] == 42
    assert returns["log"] == []

def test_terminated():
    """Calls update_state with log and raises Ignore"""
    job = Job()
    job.begin()

    def update_state(self, state, meta):
        assert state == "TERMINATED"
        assert "log" in meta

    with mock.patch(
        "celery.Task.update_state", new=update_state,
    ):
        with pytest.raises(Ignore):
            job.terminated()

def test_do_unimplemented():
    """Derived classes must implement do()."""
    job = Job()
    with pytest.raises(NotImplementedError) as excinfo:
        job.run()
    assert "Method do() is not implemented" in str(excinfo.value)
<commit_msg>test(Worker): Fix tests<commit_after>from unittest import mock

import pytest
from celery.exceptions import Ignore

from .job import Job, DEBUG, INFO, WARN, ERROR

def test_logging():
    """On each log entry update_state is called with the log."""
    job = Job()
    job.begin(task_id=4321)

    current = {}

    def send_event(self, event, **kwargs):
        assert event == "task-logged"
        current["state"] = kwargs.get("state")
        current["log"] = kwargs.get("log")

    with mock.patch(
        "celery.Task.send_event", new=send_event,
    ):

        for index, level in enumerate(["error", "warn", "info", "debug"]):
            getattr(job, level)("{} message".format(level))

            log = job.log_entries[-1]
            assert "time" in log
            assert log["level"] == index
            assert log["message"] == "{} message".format(level)
            assert current["state"] == "RUNNING"
            assert current["log"] == job.log_entries

def test_success():
    """Returns both result and log."""
    job = Job()
    job.begin()
    returns = job.success(42)
    assert returns["result"] == 42
    assert returns["log"] == []

def test_terminated():
    """Calls flush"""
    job = Job()
    job.begin()
    job.terminated()

def test_do_unimplemented():
    """Derived classes must implement do()."""
    job = Job()
    with pytest.raises(NotImplementedError) as excinfo:
        job.run()
    assert "Method do() is not implemented" in str(excinfo.value)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import copy
import logging
from datetime import datetime
import numpy

from openquake.baselib.general import AccumDict, split_in_blocks
from openquake.baselib import parallel
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.calc.hazard_curve import classical
from openquake.hazardlib import valid
from openquake.commonlib import source, readinput
from openquake.hazardlib.sourceconverter import SourceConverter

from openquake.calculators import base
from openquake.calculators.classical import ClassicalCalculator, PSHACalculator
from openquake.calculators.ucerf_event_based import (
    UCERFSource, get_composite_source_model, UcerfFilter)
# FIXME: the counting of effective ruptures has to be revised

def convert_UCERFSource(self, node):
    """
    Converts the Ucerf Source node into an SES Control object
    """
    dirname = os.path.dirname(self.fname)  # where the source_model_file is
    source_file = os.path.join(dirname, node["filename"])
    if "startDate" in node.attrib and "investigationTime" in node.attrib:
        # Is a time-dependent model - even if rates were originally
        # poissonian
        # Verify that the source time span is the same as the TOM time span
        inv_time = float(node["investigationTime"])
        if inv_time != self.tom.time_span:
            raise ValueError("Source investigation time (%s) is not "
                             "equal to configuration investigation time "
                             "(%s)" % (inv_time, self.tom.time_span))
        start_date = datetime.strptime(node["startDate"], "%d/%m/%Y")
    else:
        start_date = None
    return UCERFSource(
        source_file,
        node["id"],
        self.tom.time_span,
        start_date,
        float(node["minMag"]),
        npd=self.convert_npdist(node),
        hdd=self.convert_hpdist(node),
        aspect=~node.ruptAspectRatio,
        upper_seismogenic_depth=~node.pointGeometry.upperSeismoDepth,
        lower_seismogenic_depth=~node.pointGeometry.lowerSeismoDepth,
        msr=valid.SCALEREL[~node.magScaleRel](),
        mesh_spacing=self.rupture_mesh_spacing,
        trt=node["tectonicRegion"])

SourceConverter.convert_UCERFSource = convert_UCERFSource

@base.calculators.add('ucerf_psha')
class UcerfPSHACalculator(PSHACalculator):
    """
    UCERF classical calculator.
    """
    core_task = classical
    is_stochastic = False

    def pre_execute(self):
        """
        parse the logic tree and source model input
        """
        logging.warn('%s is still experimental', self.__class__.__name__)
        sitecol = readinput.get_site_collection(self.oqparam)
        self.datastore['sitecol'] = self.sitecol = sitecol
        self.csm = get_composite_source_model(self.oqparam)
        self.gsims_by_grp = {grp.id: self.csm.info.get_gsims(grp.id)
                             for sm in self.csm.source_models
                             for grp in sm.src_groups}
        self.rup_data = {}

    def execute(self):
        """
        Run in parallel `core_task(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor(self.core_task.__name__)
        oq = self.oqparam
        self.src_filter = UcerfFilter(self.sitecol, oq.maximum_distance)
        self.nsites = []
        acc = AccumDict({
            grp_id: ProbabilityMap(len(oq.imtls.array), len(gsims))
            for grp_id, gsims in self.gsims_by_grp.items()})
        acc.calc_times = {}
        acc.eff_ruptures = AccumDict()  # grp_id -> eff_ruptures
        acc.bb_dict = {}  # just for API compatibility
        param = dict(imtls=oq.imtls, truncation_level=oq.truncation_level,
                     filter_distance=oq.filter_distance)
        for sm in self.csm.source_models:  # one branch at the time
            grp_id = sm.ordinal
            gsims = self.gsims_by_grp[grp_id]
            [[ucerf_source]] = sm.src_groups
            self.csm.infos[ucerf_source.source_id] = source.SourceInfo(
                ucerf_source)
            ct = self.oqparam.concurrent_tasks or 1

            # parallelize by rupture subsets
            allargs = []
            for rupset_idx in split_in_blocks(
                    numpy.arange(ucerf_source.num_ruptures), ct):
                grp = copy.copy(ucerf_source)
                grp.rupset_idx = rupset_idx
                grp.num_ruptures = len(rupset_idx)
                allargs.append((grp, self.src_filter, gsims, param, monitor))
            taskname = 'ucerf_classical_%d' % grp_id
            acc = parallel.Starmap(
                classical, allargs, name=taskname
            ).reduce(self.agg_dicts, acc)

            # parallelize on the background sources, small tasks
            bckgnd_sources = ucerf_source.get_background_sources(
                self.src_filter)
            args = (bckgnd_sources, self.src_filter, gsims, param, monitor)
            bg_res = parallel.Starmap.apply(
                classical, args, name='background_sources_%d' % grp_id,
                concurrent_tasks=ct)
            # compose probabilities from background sources
            for pmap in bg_res:
                acc[grp_id] |= pmap[grp_id]

        with self.monitor('store source_info', autoflush=True):
            self.store_source_info(self.csm.infos, acc)
        return acc  # {grp_id: pmap}

@base.calculators.add('ucerf_classical')
class UCERFClassicalCalculator(ClassicalCalculator):
    pre_calculator = 'ucerf_psha'
<commit_msg>Cleanup [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import copy
import logging
from datetime import datetime
import numpy

from openquake.baselib.general import AccumDict, split_in_blocks
from openquake.baselib import parallel
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.calc.hazard_curve import classical
from openquake.hazardlib import valid
from openquake.commonlib import source, readinput
from openquake.hazardlib.sourceconverter import SourceConverter

from openquake.calculators import base
from openquake.calculators.classical import ClassicalCalculator, PSHACalculator
from openquake.calculators.ucerf_event_based import (
    UCERFSource, get_composite_source_model, UcerfFilter)
# FIXME: the counting of effective ruptures has to be revised

def convert_UCERFSource(self, node):
    """
    Converts the Ucerf Source node into an SES Control object
    """
    dirname = os.path.dirname(self.fname)  # where the source_model_file is
    source_file = os.path.join(dirname, node["filename"])
    if "startDate" in node.attrib and "investigationTime" in node.attrib:
        # Is a time-dependent model - even if rates were originally
        # poissonian
        # Verify that the source time span is the same as the TOM time span
        inv_time = float(node["investigationTime"])
        if inv_time != self.tom.time_span:
            raise ValueError("Source investigation time (%s) is not "
                             "equal to configuration investigation time "
                             "(%s)" % (inv_time, self.tom.time_span))
        start_date = datetime.strptime(node["startDate"], "%d/%m/%Y")
    else:
        start_date = None
    return UCERFSource(
        source_file,
        node["id"],
        self.tom.time_span,
        start_date,
        float(node["minMag"]),
        npd=self.convert_npdist(node),
        hdd=self.convert_hpdist(node),
        aspect=~node.ruptAspectRatio,
        upper_seismogenic_depth=~node.pointGeometry.upperSeismoDepth,
        lower_seismogenic_depth=~node.pointGeometry.lowerSeismoDepth,
        msr=valid.SCALEREL[~node.magScaleRel](),
        mesh_spacing=self.rupture_mesh_spacing,
        trt=node["tectonicRegion"])

SourceConverter.convert_UCERFSource = convert_UCERFSource

@base.calculators.add('ucerf_psha')
class UcerfPSHACalculator(PSHACalculator):
    """
    UCERF classical calculator.
    """
    core_task = classical
    is_stochastic = False

    def pre_execute(self):
        """
        parse the logic tree and source model input
        """
        logging.warn('%s is still experimental', self.__class__.__name__)
        sitecol = readinput.get_site_collection(self.oqparam)
        self.datastore['sitecol'] = self.sitecol = sitecol
        self.csm = get_composite_source_model(self.oqparam)
        self.gsims_by_grp = {grp.id: self.csm.info.get_gsims(grp.id)
                             for sm in self.csm.source_models
                             for grp in sm.src_groups}
        self.rup_data = {}

    def execute(self):
        """
        Run in parallel `core_task(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor(self.core_task.__name__)
        oq = self.oqparam
        self.src_filter = UcerfFilter(self.sitecol, oq.maximum_distance)
        self.nsites = []
        acc = AccumDict({
            grp_id: ProbabilityMap(len(oq.imtls.array), len(gsims))
            for grp_id, gsims in self.gsims_by_grp.items()})
        acc.calc_times = {}
        acc.eff_ruptures = AccumDict()  # grp_id -> eff_ruptures
        acc.bb_dict = {}  # just for API compatibility
        param = dict(imtls=oq.imtls, truncation_level=oq.truncation_level,
                     filter_distance=oq.filter_distance)
        for sm in self.csm.source_models:  # one branch at the time
            grp_id = sm.ordinal
            gsims = self.gsims_by_grp[grp_id]
            [[ucerf_source]] = sm.src_groups
            self.csm.infos[ucerf_source.source_id] = source.SourceInfo(
                ucerf_source)
            ct = self.oqparam.concurrent_tasks or 1

            # parallelize by rupture subsets
            allargs = []
            for rupset_idx in split_in_blocks(
                    numpy.arange(ucerf_source.num_ruptures), ct):
                grp = copy.copy(ucerf_source)
                grp.rupset_idx = rupset_idx
                grp.num_ruptures = len(rupset_idx)
                allargs.append((grp, self.src_filter, gsims, param, monitor))
            taskname = 'ucerf_classical_%d' % grp_id
            acc = parallel.Starmap(
                classical, allargs, name=taskname
            ).reduce(self.agg_dicts, acc)

            # parallelize on the background sources, small tasks
            bg_sources = ucerf_source.get_background_sources(self.src_filter)
            args = (bg_sources, self.src_filter, gsims, param, monitor)
            bg_res = parallel.Starmap.apply(
                classical, args, name='background_sources_%d' % grp_id,
                concurrent_tasks=ct)
            # compose probabilities
            for pmap in bg_res:
                acc[grp_id] |= pmap[grp_id]

        with self.monitor('store source_info', autoflush=True):
            self.store_source_info(self.csm.infos, acc)
        return acc  # {grp_id: pmap}

@base.calculators.add('ucerf_classical')
class UCERFClassicalCalculator(ClassicalCalculator):
    pre_calculator = 'ucerf_psha'
<|endoftext|>


<commit_before>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2012 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from django.db.models import get_model
from django.template.defaultfilters import register
from django.conf import settings
from dialer_campaign.constants import CAMPAIGN_STATUS
from dialer_campaign.views import make_duplicate_campaign
from dialer_cdr.constants import LEG_TYPE
from voice_app.constants import VOICEAPP_TYPE
from dialer_campaign.function_def import get_campaign_status_name
from dialer_campaign.views import get_campaign_survey_view, get_url_campaign_status
import os.path

@register.simple_tag(name='percentage_tag')
def percentage_tag(fraction, population):
    """Usage: {% percentage_tag fraction population %}"""
    try:
        return "%.2f%%" % ((float(fraction) / float(population)) * 100)
    except:
        return "0.00%"

@register.filter(name='contact_status')
def contact_status(value):
    """Contact status

    >>> contact_status(1)
    'ACTIVE'

    >>> contact_status(2)
    'INACTIVE'
    """
    if value == 1:
        return str('ACTIVE')
    else:
        return str('INACTIVE')

@register.filter(name='leg_type_name')
def leg_type_name(value):
    """leg type

    >>> leg_type_name(1)
    u'A-Leg'

    >>> leg_type_name(2)
    u'B-Leg'

    >>> leg_type_name(0)
    ''
    """
    if not value:
        return ''
    TYPE = dict(LEG_TYPE)
    try:
        status = TYPE[value]
    except:
        status = ''

    return unicode(status)

@register.filter(name='voiceapp_type')
def voiceapp_type(value):
    """
    >>> voiceapp_type(1)
    'DIAL'
    """
    if not value:
        return ''
    TYPE = dict(VOICEAPP_TYPE)
    try:
        status = TYPE[value]
    except:
        status = ''

    return str(status)

@register.filter(name='running_total')
def running_total(running_list, field_name):
    return sum(d[field_name] for d in running_list)

@register.filter(name='get_file_basename')
def get_file_basename(val):
    if val:
        file_url = settings.MEDIA_URL + str(val)
        return os.path.basename(file_url)
    return ''
<commit_msg>removed unwanted imports from dialer_cdr_extras<commit_after>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2012 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from django.template.defaultfilters import register
from django.conf import settings
from dialer_cdr.constants import LEG_TYPE
from voice_app.constants import VOICEAPP_TYPE
import os.path

@register.simple_tag(name='percentage_tag')
def percentage_tag(fraction, population):
    """Usage: {% percentage_tag fraction population %}"""
    try:
        return "%.2f%%" % ((float(fraction) / float(population)) * 100)
    except:
        return "0.00%"

@register.filter(name='contact_status')
def contact_status(value):
    """Contact status

    >>> contact_status(1)
    'ACTIVE'

    >>> contact_status(2)
    'INACTIVE'
    """
    if value == 1:
        return str('ACTIVE')
    else:
        return str('INACTIVE')

@register.filter(name='leg_type_name')
def leg_type_name(value):
    """leg type

    >>> leg_type_name(1)
    u'A-Leg'

    >>> leg_type_name(2)
    u'B-Leg'

    >>> leg_type_name(0)
    ''
    """
    if not value:
        return ''
    TYPE = dict(LEG_TYPE)
    try:
        status = TYPE[value]
    except:
        status = ''

    return unicode(status)

@register.filter(name='voiceapp_type')
def voiceapp_type(value):
    """
    >>> voiceapp_type(1)
    'DIAL'
    """
    if not value:
        return ''
    TYPE = dict(VOICEAPP_TYPE)
    try:
        status = TYPE[value]
    except:
        status = ''

    return str(status)

@register.filter(name='running_total')
def running_total(running_list, field_name):
    return sum(d[field_name] for d in running_list)

@register.filter(name='get_file_basename')
def get_file_basename(val):
    if val:
        file_url = settings.MEDIA_URL + str(val)
        return os.path.basename(file_url)
    return ''
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
from datetime import datetime
from zipfile import ZipFile
from couchdbkit.exceptions import ResourceNotFound, BadValueError
from corehq.apps.app_manager.const import APP_V2
from dimagi.ext.couchdbkit import *
from corehq.apps.builds.fixtures import commcare_build_config
from corehq.apps.builds.jadjar import JadJar
from corehq.util.quickcache import quickcache
from itertools import groupby
from distutils.version import LooseVersion

class SemanticVersionProperty(StringProperty):

    def validate(self, value, required=True):
        super(SemanticVersionProperty, self).validate(value, required)
        try:
            major, minor, _ = value.split('.')
            int(major)
            int(minor)
        except Exception:
            raise BadValueError("Build version %r does not comply with the x.y.z schema" % value)
        return value

class CommCareBuild(Document):
    """
    #python manage.py shell
    #>>> from corehq.apps.builds.models import CommCareBuild
    #>>> build = CommCareBuild.create_from_zip('/Users/droberts/Desktop/zip/7106.zip', '1.2.dev', 7106)

    """

    build_number = IntegerProperty()
    version = SemanticVersionProperty()
    time = DateTimeProperty()
    j2me_enabled = BooleanProperty(default=True)

    def put_file(self, payload, path, filename=None):
        """
        Add an attachment to the build (useful for constructing the build)
        payload should be a file-like object
        filename should be something like "Nokia/S40-generic/CommCare.jar"

        """
        if filename:
            path = '/'.join([path, filename])
        content_type = {
            'jad': 'text/vnd.sun.j2me.app-descriptor',
            'jar': 'application/java-archive',
        }.get(path.split('.')[-1])
        self.put_attachment(payload, path, content_type)

    def fetch_file(self, path, filename=None):
        if filename:
            path = '/'.join([path, filename])
        return self.fetch_attachment(path)

    def get_jadjar(self, path, use_j2me_endpoint):
        """
        build.get_jadjar("Nokia/S40-generic")
        """
        try:
            jad = self.fetch_file(path, "CommCare.jad")
        except ResourceNotFound:
            jad = None

        return JadJar(
            jad=jad,
            jar=self.fetch_file(path, "CommCare.jar"),
            version=self.version,
            build_number=self.build_number,
            use_j2me_endpoint=use_j2me_endpoint,
        )

    @classmethod
    def create_from_zip(cls, f, version, build_number):
        """f should be a file-like object or a path to a zipfile"""
        self = cls(build_number=build_number, version=version, time=datetime.utcnow())
        self.save()
        # Clear cache to have this new build included immediately
        cls.j2me_enabled_builds.clear(cls)

        with ZipFile(f) as z:
            try:
                for name in z.namelist():
                    path = name.split('/')
                    if path[0] == "dist" and path[-1] != "":
                        path = '/'.join(path[1:])
                        self.put_file(z.read(name), path)
            except:
                self.delete()
                raise
        return self

    @classmethod
    def create_without_artifacts(cls, version, build_number):
        self = cls(build_number=build_number, version=version,
                   time=datetime.utcnow(), j2me_enabled=False)
        self.save()
        # Clear cache to have this build number excluded immediately if build added
        # with existing version number but not supporting j2me now
        cls.j2me_enabled_builds.clear(cls)
        return self

    def minor_release(self):
        major, minor, _ = self.version.split('.')
        return int(major), int(minor)

    def major_release(self):
        major, _, _ = self.version.split('.')
        return int(major)

    @classmethod
    def get_build(cls, version, build_number=None, latest=False):
        """
        Call as either
            CommCareBuild.get_build(version, build_number)
        or
            CommCareBuild.get_build(version, latest=True)
        """

        if latest:
            startkey = [version]
        else:
            build_number = int(build_number)
            startkey = [version, build_number]

        self = cls.view('builds/all',
            startkey=startkey + [{}],
            endkey=startkey,
            descending=True,
            limit=1,
            include_docs=True,
            reduce=False,
        ).one()

        if not self:
            raise KeyError("Can't find build {label}. For instructions on how to add it, see https://github.com/dimagi/commcare-hq/blob/master/corehq/apps/builds/README.md".format(label=BuildSpec(
                version=version,
                build_number=build_number,
                latest=latest
            )))
        return self

    @classmethod
    def all_builds(cls):
        return cls.view('builds/all', include_docs=True, reduce=False)

    @classmethod
    @quickcache([], timeout=5 * 60)
    def j2me_enabled_builds(cls):
        j2me_enabled_builds = []
        for version_number, group in groupby(cls.all_builds(), lambda x: x['version']):
            latest_version_build = list(group)[-1]
            if latest_version_build['j2me_enabled']:
                j2me_enabled_builds.append(latest_version_build)

        return j2me_enabled_builds

    @classmethod
    def j2me_enabled_build_versions(cls):
        return [x.version for x in cls.j2me_enabled_builds()]

class BuildSpec(DocumentSchema):
    version = StringProperty()
    build_number = IntegerProperty(required=False)
    latest = BooleanProperty()

    def get_build(self):
        if self.latest:
            return CommCareBuild.get_build(self.version, latest=True)
        else:
            return CommCareBuild.get_build(self.version, self.build_number)

    def is_null(self):
        return not (self.version and (self.build_number or self.latest))

    def get_label(self):
        if not self.is_null():
            fmt = "{self.version}"
            return fmt.format(self=self)
        else:
            return None

    def get_menu_item_label(self):
        build_config = CommCareBuildConfig.fetch()
        for item in build_config.menu:
            if item.build.version == self.version:
                return item.label or self.get_label()
        return self.get_label()

    def supports_j2me(self):
        return self.get_menu_item_label() in CommCareBuildConfig.j2me_enabled_config_labels()

    def __str__(self):
        fmt = "{self.version}/"
        fmt += "latest" if self.latest else "{self.build_number}"
        return fmt.format(self=self)

    def to_string(self):
        return str(self)

    @classmethod
    def from_string(cls, string):
        version, build_number = string.split('/')
        if build_number == "latest":
            return cls(version=version, latest=True)
        else:
            build_number = int(build_number)
            return cls(version=version, build_number=build_number)

    def minor_release(self):
        major, minor, _ = self.version.split('.')
        return int(major), int(minor)

    def major_release(self):
        return self.version.split('.')[0]

    def release_greater_than_or_equal_to(self, version):
        if not self.version:
            return False
        return LooseVersion(self.version) >= LooseVersion(version)

class BuildMenuItem(DocumentSchema):
    build = SchemaProperty(BuildSpec)
    label = StringProperty(required=False)
    superuser_only = BooleanProperty(default=False)
    j2me_enabled = BooleanProperty(default=True)

    def get_build(self):
        return self.build.get_build()

    def get_label(self):
        return self.label or self.build.get_label()

class CommCareBuildConfig(Document):
    _ID = 'config--commcare-builds'

    preview = SchemaProperty(BuildSpec)
    defaults = SchemaListProperty(BuildSpec)
    application_versions = StringListProperty()
    menu = SchemaListProperty(BuildMenuItem)

    @classmethod
    def bootstrap(cls):
        config = cls.wrap(commcare_build_config)
        config._id = config._ID
        config.save()
        return config

    @classmethod
    def clear_local_cache(cls):
        cls.fetch.clear(cls)

    @classmethod
    @quickcache([], timeout=24 * 60 * 60)
    def fetch(cls):
        try:
            return cls.get(cls._ID)
        except ResourceNotFound:
            return cls.bootstrap()

    def get_default(self, application_version=APP_V2):
        i = self.application_versions.index(application_version)
        return self.defaults[i]

    def get_menu(self):
        return self.menu

    @classmethod
    @quickcache([], timeout=5 * 60)
    #This seems to be not working.
    def j2me_enabled_configs(cls):
        return [build for build in cls.fetch().menu if build.j2me_enabled]

    @classmethod
    def j2me_enabled_config_labels(cls):
        return [x.label for x in cls.j2me_enabled_configs()]

    @classmethod
    def latest_j2me_enabled_config(cls):
        return cls.j2me_enabled_configs()[-1]

class BuildRecord(BuildSpec):
    signed = BooleanProperty(default=True)
    datetime = DateTimeProperty(required=False)
<commit_msg>use StrictVersion instead of LooseVersion<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals
from datetime import datetime
from zipfile import ZipFile
from couchdbkit.exceptions import ResourceNotFound, BadValueError
from corehq.apps.app_manager.const import APP_V2
from dimagi.ext.couchdbkit import *
from corehq.apps.builds.fixtures import commcare_build_config
from corehq.apps.builds.jadjar import JadJar
from corehq.util.quickcache import quickcache
from itertools import groupby
from distutils.version import StrictVersion

class SemanticVersionProperty(StringProperty):

    def validate(self, value, required=True):
        super(SemanticVersionProperty, self).validate(value, required)
        try:
            major, minor, _ = value.split('.')
            int(major)
            int(minor)
        except Exception:
            raise BadValueError("Build version %r does not comply with the x.y.z schema" % value)
        return value

class CommCareBuild(Document):
    """
    #python manage.py shell
    #>>> from corehq.apps.builds.models import CommCareBuild
    #>>> build = CommCareBuild.create_from_zip('/Users/droberts/Desktop/zip/7106.zip', '1.2.dev', 7106)

    """

    build_number = IntegerProperty()
    version = SemanticVersionProperty()
    time = DateTimeProperty()
    j2me_enabled = BooleanProperty(default=True)

    def put_file(self, payload, path, filename=None):
        """
        Add an attachment to the build (useful for constructing the build)
        payload should be a file-like object
        filename should be something like "Nokia/S40-generic/CommCare.jar"

        """
        if filename:
            path = '/'.join([path, filename])
        content_type = {
            'jad': 'text/vnd.sun.j2me.app-descriptor',
            'jar': 'application/java-archive',
        }.get(path.split('.')[-1])
        self.put_attachment(payload, path, content_type)

    def fetch_file(self, path, filename=None):
        if filename:
            path = '/'.join([path, filename])
        return self.fetch_attachment(path)

    def get_jadjar(self, path, use_j2me_endpoint):
        """
        build.get_jadjar("Nokia/S40-generic")
        """
        try:
            jad = self.fetch_file(path, "CommCare.jad")
        except ResourceNotFound:
            jad = None

        return JadJar(
            jad=jad,
            jar=self.fetch_file(path, "CommCare.jar"),
            version=self.version,
            build_number=self.build_number,
            use_j2me_endpoint=use_j2me_endpoint,
        )

    @classmethod
    def create_from_zip(cls, f, version, build_number):
        """f should be a file-like object or a path to a zipfile"""
        self = cls(build_number=build_number, version=version, time=datetime.utcnow())
        self.save()
        # Clear cache to have this new build included immediately
        cls.j2me_enabled_builds.clear(cls)

        with ZipFile(f) as z:
            try:
                for name in z.namelist():
                    path = name.split('/')
                    if path[0] == "dist" and path[-1] != "":
                        path = '/'.join(path[1:])
                        self.put_file(z.read(name), path)
            except:
                self.delete()
                raise
        return self

    @classmethod
    def create_without_artifacts(cls, version, build_number):
        self = cls(build_number=build_number, version=version,
                   time=datetime.utcnow(), j2me_enabled=False)
        self.save()
        # Clear cache to have this build number excluded immediately if build added
        # with existing version number but not supporting j2me now
        cls.j2me_enabled_builds.clear(cls)
        return self

    def minor_release(self):
        major, minor, _ = self.version.split('.')
        return int(major), int(minor)

    def major_release(self):
        major, _, _ = self.version.split('.')
        return int(major)

    @classmethod
    def get_build(cls, version, build_number=None, latest=False):
        """
        Call as either
            CommCareBuild.get_build(version, build_number)
        or
            CommCareBuild.get_build(version, latest=True)
        """

        if latest:
            startkey = [version]
        else:
            build_number = int(build_number)
            startkey = [version, build_number]

        self = cls.view('builds/all',
            startkey=startkey + [{}],
            endkey=startkey,
            descending=True,
            limit=1,
            include_docs=True,
            reduce=False,
        ).one()

        if not self:
            raise KeyError("Can't find build {label}. For instructions on how to add it, see https://github.com/dimagi/commcare-hq/blob/master/corehq/apps/builds/README.md".format(label=BuildSpec(
                version=version,
                build_number=build_number,
                latest=latest
            )))
        return self

    @classmethod
    def all_builds(cls):
        return cls.view('builds/all', include_docs=True, reduce=False)

    @classmethod
    @quickcache([], timeout=5 * 60)
    def j2me_enabled_builds(cls):
        j2me_enabled_builds = []
        for version_number, group in groupby(cls.all_builds(), lambda x: x['version']):
            latest_version_build = list(group)[-1]
            if latest_version_build['j2me_enabled']:
                j2me_enabled_builds.append(latest_version_build)

        return j2me_enabled_builds

    @classmethod
    def j2me_enabled_build_versions(cls):
        return [x.version for x in cls.j2me_enabled_builds()]

class BuildSpec(DocumentSchema):
    version = StringProperty()
    build_number = IntegerProperty(required=False)
    latest = BooleanProperty()

    def get_build(self):
        if self.latest:
            return CommCareBuild.get_build(self.version, latest=True)
        else:
            return CommCareBuild.get_build(self.version, self.build_number)

    def is_null(self):
        return not (self.version and (self.build_number or self.latest))

    def get_label(self):
        if not self.is_null():
            fmt = "{self.version}"
            return fmt.format(self=self)
        else:
            return None

    def get_menu_item_label(self):
        build_config = CommCareBuildConfig.fetch()
        for item in build_config.menu:
            if item.build.version == self.version:
                return item.label or self.get_label()
        return self.get_label()

    def supports_j2me(self):
        return self.get_menu_item_label() in CommCareBuildConfig.j2me_enabled_config_labels()

    def __str__(self):
        fmt = "{self.version}/"
        fmt += "latest" if self.latest else "{self.build_number}"
        return fmt.format(self=self)

    def to_string(self):
        return str(self)

    @classmethod
    def from_string(cls, string):
        version, build_number = string.split('/')
        if build_number == "latest":
            return cls(version=version, latest=True)
        else:
            build_number = int(build_number)
            return cls(version=version, build_number=build_number)

    def minor_release(self):
        major, minor, _ = self.version.split('.')
        return int(major), int(minor)

    def major_release(self):
        return self.version.split('.')[0]

    def release_greater_than_or_equal_to(self, version):
        if not self.version:
            return False
        return StrictVersion(self.version) >= StrictVersion(version)

class BuildMenuItem(DocumentSchema):
    build = SchemaProperty(BuildSpec)
    label = StringProperty(required=False)
    superuser_only = BooleanProperty(default=False)
    j2me_enabled = BooleanProperty(default=True)

    def get_build(self):
        return self.build.get_build()

    def get_label(self):
        return self.label or self.build.get_label()

class CommCareBuildConfig(Document):
    _ID = 'config--commcare-builds'

    preview = SchemaProperty(BuildSpec)
    defaults = SchemaListProperty(BuildSpec)
    application_versions = StringListProperty()
    menu = SchemaListProperty(BuildMenuItem)

    @classmethod
    def bootstrap(cls):
        config = cls.wrap(commcare_build_config)
        config._id = config._ID
        config.save()
        return config

    @classmethod
    def clear_local_cache(cls):
        cls.fetch.clear(cls)

    @classmethod
    @quickcache([], timeout=24 * 60 * 60)
    def fetch(cls):
        try:
            return cls.get(cls._ID)
        except ResourceNotFound:
            return cls.bootstrap()

    def get_default(self, application_version=APP_V2):
        i = self.application_versions.index(application_version)
        return self.defaults[i]

    def get_menu(self):
        return self.menu

    @classmethod
    @quickcache([], timeout=5 * 60)
    #This seems to be not working.
    def j2me_enabled_configs(cls):
        return [build for build in cls.fetch().menu if build.j2me_enabled]

    @classmethod
    def j2me_enabled_config_labels(cls):
        return [x.label for x in cls.j2me_enabled_configs()]

    @classmethod
    def latest_j2me_enabled_config(cls):
        return cls.j2me_enabled_configs()[-1]

class BuildRecord(BuildSpec):
    signed = BooleanProperty(default=True)
    datetime = DateTimeProperty(required=False)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Given a number of BAM files, this script will merge them together, taking
only the best matches.  There can be any number of files, but the BAM header
will be taken from the first one.  The input files should be sorted by read
name, or at least have reads in the same order.

The first input file should have a record for every read in the other files.
However, the secondary files *may* have missing lines, so long as they are in
the same order as the first file.
"""

import os,sys,gzip
from support.eta import ETA
import pysam

def usage():
    base = os.path.basename(sys.argv[0])
    print __doc__
    print """
Usage: %s out.bam in1.bam in2.bam ...

""" % (base,)
    sys.exit(1)

def bam_reads_batch(bam):
    reads = []
    last = None
    for read in bam:
        if last and read.qname != last:
            yield reads
            reads = []
        last = read.qname
        reads.append(read)

    if reads:
        yield reads

def bam_merge(fname,infiles,tag='AS'):
    bams = []
    last_reads = []
    bamgens = []

    for infile in infiles:
        bam=pysam.Samfile(infile,"rb")
        last_reads.append(None)
        bams.append(bam)
        bamgens.append(bam_reads_batch(bam))

    outfile = pysam.Samfile(fname,"wb",template=bams[0])
    i = 0

    while True:
        found = False
        for i,bamgen in enumerate(bamgens):
            if last_reads[i] == None:
                try:
                    last_reads[i] = bamgen.next()
                    if last_reads[i]:
                        found = True
                except:
                    pass
            else:
                found = True
        if not found:
            break

        best_val = None
        best_reads = None

        # for fn,reads in zip(infiles,last_reads):
        #     print os.path.basename(fn),reads[0].qname,reads[0].is_unmapped,reads[0].opt(tag)
        # print
        #
        for i in xrange(len(last_reads)):
            if not last_reads[i]:
                continue

            for read in last_reads[i].reads:
                if read.qname == last_reads[0][0].qname:
                    if not read.is_unmapped:
                        tag_val = int(read.opt(tag))
                        if not best_val or tag_val > best_val:
                            best_val = tag_val
                            best_reads = last_reads[i]
                            break
                    last_reads[i]=None

        if best_reads:
            for read in best_reads:
                outfile.write(read)
        i += 1
        if i > 100:
            return

    eta.done()
    outfile.close()
    for bam in bams:
        bam.close()

if __name__ == '__main__':
    infiles = []
    outfile = None
    num=1000000
    last = None

    for arg in sys.argv[1:]:
        if arg == '-h':
            usage()
        elif not outfile:
            outfile = arg
        elif os.path.exists(arg):
            infiles.append(arg)

    if not infiles or not outfile:
        usage()
    else:
        bam_merge(outfile,infiles)

<commit_msg>Updated merge to actually work.<commit_after>#!/usr/bin/env python
"""
Given a number of BAM files, this script will merge them together, taking
only the best matches.  There can be any number of files, but the BAM header
will be taken from the first one.  The input files should be sorted by read
name, or at least have reads in the same order.

The first input file should have a record for every read in the other files.
However, the secondary files *may* have missing lines, so long as they are in
the same order as the first file.

The value of the attribute/tag given will be used to determine which reads
should be kept and which should be discarded. The tag should be a numberic
(int/float) type. This defaults to 'AS'.

Additionally, each file can have more than one record for each read, but they
should all have the same value for the tag used in determining which reads to
keep. For example, if the AS tag is used (default), then each read in a file
should have the same AS value. Reads in different files will have different
values.

"""

import os,sys,gzip
import pysam

def usage():
    base = os.path.basename(sys.argv[0])
    print __doc__
    print """
Usage: %s {-tag VAL} out.bam in1.bam in2.bam ...

""" % (base,)
    sys.exit(1)

def bam_reads_batch(bam):
    reads = []
    last = None
    for read in bam:
        if last and read.qname != last:
            yield reads
            reads = []
        last = read.qname
        reads.append(read)

    if reads:
        yield reads

def bam_merge(fname,infiles,tag='AS'):
    bams = []
    last_reads = []
    bamgens = []
    counts = []
    unmapped = 0

    for infile in infiles:
        bam=pysam.Samfile(infile,"rb")
        last_reads.append(None)
        bams.append(bam)
        counts.append(0)
        bamgens.append(bam_reads_batch(bam))

    outfile = pysam.Samfile(fname,"wb",template=bams[0])

    while True:
        found = False
        for i,bamgen in enumerate(bamgens):
            if last_reads[i] == None:
#                print 'loading from %s' % infiles[i]
                try:
                    last_reads[i] = bamgen.next()
                    if last_reads[i]:
                        found = True
                except:
                    pass
            else:
                found = True
        if not found:
            break

        best_val = None
        best_reads = None
        best_source = 0

#        for fn,reads in zip(infiles,last_reads):
#            print os.path.basename(fn),reads[0].qname,reads[0].is_unmapped,reads[0].opt(tag)
#        print
#        print last_reads[0][0].qname

        first_qname = last_reads[0][0].qname
        for i in xrange(len(last_reads)):
            if not last_reads[i]:
                continue

            match = False
            for read in last_reads[i]:
                if read.qname == first_qname:
                    match = True
                    if not read.is_unmapped:
                        tag_val = int(read.opt(tag))
                        if not best_val or tag_val > best_val:
                            best_val = tag_val
                            best_reads = last_reads[i]
                            best_source = i
                            break
            if match:
                last_reads[i]=None

        if best_reads:
#            print infiles[best_source],best_reads[0].qname
            counts[best_source] += 1
            for read in best_reads:
                outfile.write(read)
        else:
            unmapped += 1

    for fn,cnt in zip(infiles,counts):
        print "%s\t%s" % (fn,cnt)
    print "unmapped\t%s" % unmapped

    outfile.close()
    for bam in bams:
        bam.close()

if __name__ == '__main__':
    infiles = []
    outfile = None
    last = None
    tag = 'AS'

    for arg in sys.argv[1:]:
        if arg == '-h':
            usage()
        elif last == '-tag':
            tag = arg
            last = None
        elif arg in ['-tag']:
            last = arg
        elif not outfile:
            outfile = arg
        elif os.path.exists(arg):
            infiles.append(arg)

    if not infiles or not outfile:
        usage()
    else:
        bam_merge(outfile,infiles,tag)

<|endoftext|>