<commit_before># -*- coding: utf-8 -*-

from __future__ import division

from collections import Iterable
import itertools

from ..translator import _
from ..exceptions import ValidationError, ConversionError, ModelValidationError, StopValidation
from ..models import Model
from ..transforms import export_loop, EMPTY_LIST, EMPTY_DICT
from .base import BaseType

from six import iteritems
from six import string_types as basestring
from six import text_type as unicode

class MultiType(BaseType):

    def validate(self, value):
        """Report dictionary of errors with lists of errors as values of each
        key. Used by ModelType and ListType.

        """
        errors = {}

        for validator in self.validators:
            try:
                validator(value)
            except ModelValidationError as exc:
                errors.update(exc.messages)
            except StopValidation as exc:
                errors.update(exc.messages)
                break

        if errors:
            raise ValidationError(errors)

        return value

    def export_loop(self, shape_instance, field_converter,
                    role=None, print_none=False):
        raise NotImplementedError

    def init_compound_field(self, field, compound_field, **kwargs):
        """
        Some of non-BaseType fields requires `field` arg.
        Not avoid name conflict, provide it as `compound_field`.
        Example:

            comments = ListType(DictType, compound_field=StringType)
        """
        if compound_field:
            field = field(field=compound_field, **kwargs)
        else:
            field = field(**kwargs)
        return field

class ModelType(MultiType):

    MESSAGES = {
        'use_mapping': _(u"Please use a mapping for this field or {0} instance instead of {1}."),
    }

    def __init__(self, model_class, **kwargs):
        self.model_class = model_class
        self.fields = self.model_class.fields

        validators = kwargs.pop("validators", [])
        self.strict = kwargs.pop("strict", True)

        def validate_model(model_instance):
            model_instance.validate()
            return model_instance

        super(ModelType, self).__init__(validators=[validate_model] + validators, **kwargs)

    def __repr__(self):
        return object.__repr__(self)[:-1] + ' for %s>' % self.model_class

    def to_native(self, value, mapping=None, context=None):
        # We have already checked if the field is required. If it is None it
        # should continue being None
        if mapping is None:
            mapping = {}
        if value is None:
            return None
        if isinstance(value, self.model_class):
            return value

        if not isinstance(value, dict):
            raise ConversionError(
                self.messages['use_mapping'].format(
                    self.model_class.__name__,
                    type(value).__name__))

        # partial submodels now available with import_data (ht ryanolson)
        model = self.model_class()
        return model.import_data(value, mapping=mapping, context=context,
                                 strict=self.strict)

    def to_primitive(self, model_instance, context=None):
        primitive_data = {}
        for field_name, field, value in model_instance.atoms():
            serialized_name = field.serialized_name or field_name

            if value is None and model_instance.allow_none(field):
                primitive_data[serialized_name] = None
            else:
                primitive_data[serialized_name] = field.to_primitive(value,
                                                                     context)

        return primitive_data

    def export_loop(self, model_instance, field_converter,
                    role=None, print_none=False):
        """
        Calls the main `export_loop` implementation because they are both
        supposed to operate on models.
        """
        if isinstance(model_instance, self.model_class):
            model_class = model_instance.__class__
        else:
            model_class = self.model_class

        shaped = export_loop(model_class, model_instance,
                             field_converter,
                             role=role, print_none=print_none)

        if shaped and len(shaped) == 0 and self.allow_none():
            return shaped
        elif shaped:
            return shaped
        elif print_none:
            return shaped

class ListType(MultiType):

    MESSAGES = {
        'at_least_item': _(u"Please provide at least {0} item."),
        'mote_than_item': _(u"Please provide no more than {0} item."),
    }

    def __init__(self, field, min_size=None, max_size=None, **kwargs):

        if not isinstance(field, BaseType):
            compound_field = kwargs.pop('compound_field', None)
            field = self.init_compound_field(field, compound_field, **kwargs)

        self.field = field
        self.min_size = min_size
        self.max_size = max_size

        validators = [self.check_length, self.validate_items] + kwargs.pop("validators", [])

        super(ListType, self).__init__(validators=validators, **kwargs)

    @property
    def model_class(self):
        return self.field.model_class

    def _force_list(self, value):
        if value is None or value == EMPTY_LIST:
            return []

        try:
            if isinstance(value, basestring):
                raise TypeError()

            if isinstance(value, dict):
                return [value[unicode(k)] for k in sorted(map(int, value.keys()))]

            return list(value)
        except TypeError:
            return [value]

    def to_native(self, value, context=None):
        items = self._force_list(value)

        return [self.field.to_native(item, context) for item in items]

    def check_length(self, value):
        list_length = len(value) if value else 0

        if self.min_size is not None and list_length < self.min_size:
            message = ({
                True: self.messages['at_least_item'],
                False: self.messages['at_least_item'],
            }[self.min_size == 1]).format(self.min_size)
            raise ValidationError(message)

        if self.max_size is not None and list_length > self.max_size:
            message = ({
                True: self.messages['mote_than_item'],
                False: self.messages['mote_than_item'],
            }[self.max_size == 1]).format(self.max_size)
            raise ValidationError(message)

    def validate_items(self, items):
        errors = []
        for item in items:
            try:
                self.field.validate(item)
            except ValidationError as exc:
                errors.append(exc.messages)
        if errors:
            raise ValidationError(errors)

    def to_primitive(self, value, context=None):
        return [self.field.to_primitive(item, context) for item in value]

    def export_loop(self, list_instance, field_converter,
                    role=None, print_none=False):
        """Loops over each item in the model and applies either the field
        transform or the multitype transform.  Essentially functions the same
        as `transforms.export_loop`.
        """
        data = []
        for value in list_instance:
            if hasattr(self.field, 'export_loop'):
                shaped = self.field.export_loop(value, field_converter,
                                                role=role)
                feels_empty = shaped and len(shaped) == 0
            else:
                shaped = field_converter(self.field, value)
                feels_empty = shaped is None

            # Print if we want empty or found a value
            if feels_empty and self.field.allow_none():
                data.append(shaped)
            elif shaped is not None:
                data.append(shaped)
            elif print_none:
                data.append(shaped)

        # Return data if the list contains anything
        if len(data) > 0:
            return data
        elif len(data) == 0 and self.allow_none():
            return data
        elif print_none:
            return data

class DictType(MultiType):

    MESSAGES = {
        'only_dict': _(u"Only dictionaries may be used in a DictType."),
    }

    def __init__(self, field, coerce_key=None, **kwargs):
        if not isinstance(field, BaseType):
            compound_field = kwargs.pop('compound_field', None)
            field = self.init_compound_field(field, compound_field, **kwargs)

        self.coerce_key = coerce_key or unicode
        self.field = field

        validators = [self.validate_items] + kwargs.pop("validators", [])

        super(DictType, self).__init__(validators=validators, **kwargs)

    @property
    def model_class(self):
        return self.field.model_class

    def to_native(self, value, safe=False, context=None):
        if value == EMPTY_DICT:
            value = {}

        value = value or {}

        if not isinstance(value, dict):
            raise ValidationError(self.messages['only_dict'])

        return dict((self.coerce_key(k), self.field.to_native(v, context))
                    for k, v in iteritems(value))

    def validate_items(self, items):
        errors = {}
        for key, value in iteritems(items):
            try:
                self.field.validate(value)
            except ValidationError as exc:
                errors[key] = exc

        if errors:
            raise ValidationError(errors)

    def to_primitive(self, value, context=None):
        return dict((unicode(k), self.field.to_primitive(v, context))
                    for k, v in iteritems(value))

    def export_loop(self, dict_instance, field_converter,
                    role=None, print_none=False):
        """Loops over each item in the model and applies either the field
        transform or the multitype transform.  Essentially functions the same
        as `transforms.export_loop`.
        """
        data = {}

        for key, value in iteritems(dict_instance):
            if hasattr(self.field, 'export_loop'):
                shaped = self.field.export_loop(value, field_converter,
                                                role=role)
                feels_empty = shaped and len(shaped) == 0
            else:
                shaped = field_converter(self.field, value)
                feels_empty = shaped is None

            if feels_empty and self.field.allow_none():
                data[key] = shaped
            elif shaped is not None:
                data[key] = shaped
            elif print_none:
                data[key] = shaped

        if len(data) > 0:
            return data
        elif len(data) == 0 and self.allow_none():
            return data
        elif print_none:
            return data

class PolyModelType(MultiType):

    def __init__(self, model_classes, **kwargs):

        if isinstance(model_classes, type) and issubclass(model_classes, Model):
            self.model_classes = (model_classes,)
            allow_subclasses = True
        elif isinstance(model_classes, Iterable) \
          and not isinstance(model_classes, basestring):
            self.model_classes = tuple(model_classes)
            allow_subclasses = False
        else:
            raise Exception("The first argument to PolyModelType.__init__() "
                            "must be a model or an iterable.")

        validators = kwargs.pop("validators", [])
        self.strict = kwargs.pop("strict", True)
        self.claim_function = kwargs.pop("claim_function", None)
        self.allow_subclasses = kwargs.pop("allow_subclasses", allow_subclasses)

        def validate_model(model_instance):
            model_instance.validate()
            return model_instance

        MultiType.__init__(self, validators=[validate_model] + validators, **kwargs)

    def __repr__(self):
        return object.__repr__(self)[:-1] + ' for %s>' % str(self.model_classes)

    def is_allowed_model(self, model_instance):
        if self.allow_subclasses:
            if isinstance(model_instance, self.model_classes):
                return True
        else:
            if model_instance.__class__ in self.model_classes:
                return True
        return False

    def to_native(self, value, mapping=None, context=None):

        if mapping is None:
            mapping = {}
        if value is None:
            return None
        if self.is_allowed_model(value):
            return value
        if not isinstance(value, dict):
            if len(self.model_classes) > 1:
                instanceof_msg = 'one of: {}'.format(', '.join(
                    cls.__name__ for cls in self.model_classes))
            else:
                instanceof_msg = self.model_classes[0].__name__
            raise ConversionError(u'Please use a mapping for this field or '
                                    'an instance of {}'.format(instanceof_msg))

        model_class = self.find_model(value)
        model = model_class()
        return model.import_data(value, mapping=mapping, context=context,
                                 strict=self.strict)

    def find_model(self, data):
        """Finds the intended type by consulting potential classes or `claim_function`."""

        chosen_class = None
        if self.claim_function:
            chosen_class = self.claim_function(self, data)
        else:
            candidates = self.model_classes
            if self.allow_subclasses:
                candidates = itertools.chain.from_iterable(
                                 ([m] + m._subclasses for m in candidates))
            fallback = None
            matching_classes = []
            for cls in candidates:
                match = None
                if '_claim_polymorphic' in cls.__dict__:
                    match = cls._claim_polymorphic(data)
                elif not fallback: # The first model that doesn't define the hook
                    fallback = cls # can be used as a default if there's no match.
                if match:
                    matching_classes.append(cls)
            if not matching_classes and fallback:
                chosen_class = fallback
            elif len(matching_classes) == 1:
                chosen_class = matching_classes[0]
            else:
                raise Exception("Got ambiguous input for polymorphic field")
        if chosen_class:
            return chosen_class
        else:
            raise Exception("Input for polymorphic field did not match any model")

    def export_loop(self, model_instance, field_converter,
                    role=None, print_none=False):

        model_class = model_instance.__class__
        if not self.is_allowed_model(model_instance):
            raise Exception("Cannot export: {} is not an allowed type".format(model_class))

        shaped = export_loop(model_class, model_instance,
                             field_converter,
                             role=role, print_none=print_none)

        if shaped and len(shaped) == 0 and self.allow_none():
            return shaped
        elif shaped:
            return shaped
        elif print_none:
            return shaped

<commit_msg>Fix for code review<commit_after># -*- coding: utf-8 -*-

from __future__ import division

from collections import Iterable
import itertools

from ..translator import _
from ..exceptions import ValidationError, ConversionError, ModelValidationError, StopValidation
from ..models import Model
from ..transforms import export_loop, EMPTY_LIST, EMPTY_DICT
from .base import BaseType

from six import iteritems
from six import string_types as basestring
from six import text_type as unicode

class MultiType(BaseType):

    def validate(self, value):
        """Report dictionary of errors with lists of errors as values of each
        key. Used by ModelType and ListType.

        """
        errors = {}

        for validator in self.validators:
            try:
                validator(value)
            except ModelValidationError as exc:
                errors.update(exc.messages)
            except StopValidation as exc:
                errors.update(exc.messages)
                break

        if errors:
            raise ValidationError(errors)

        return value

    def export_loop(self, shape_instance, field_converter,
                    role=None, print_none=False):
        raise NotImplementedError

    def init_compound_field(self, field, compound_field, **kwargs):
        """
        Some of non-BaseType fields requires `field` arg.
        Not avoid name conflict, provide it as `compound_field`.
        Example:

            comments = ListType(DictType, compound_field=StringType)
        """
        if compound_field:
            field = field(field=compound_field, **kwargs)
        else:
            field = field(**kwargs)
        return field

class ModelType(MultiType):

    MESSAGES = {
        'use_mapping': _(u"Please use a mapping for this field or {0} instance instead of {1}."),
    }

    def __init__(self, model_class, **kwargs):
        self.model_class = model_class
        self.fields = self.model_class.fields

        validators = kwargs.pop("validators", [])
        self.strict = kwargs.pop("strict", True)

        def validate_model(model_instance):
            model_instance.validate()
            return model_instance

        super(ModelType, self).__init__(validators=[validate_model] + validators, **kwargs)

    def __repr__(self):
        return object.__repr__(self)[:-1] + ' for %s>' % self.model_class

    def to_native(self, value, mapping=None, context=None):
        # We have already checked if the field is required. If it is None it
        # should continue being None
        if mapping is None:
            mapping = {}
        if value is None:
            return None
        if isinstance(value, self.model_class):
            return value

        if not isinstance(value, dict):
            raise ConversionError(
                self.messages['use_mapping'].format(
                    self.model_class.__name__,
                    type(value).__name__))

        # partial submodels now available with import_data (ht ryanolson)
        model = self.model_class()
        return model.import_data(value, mapping=mapping, context=context,
                                 strict=self.strict)

    def to_primitive(self, model_instance, context=None):
        primitive_data = {}
        for field_name, field, value in model_instance.atoms():
            serialized_name = field.serialized_name or field_name

            if value is None and model_instance.allow_none(field):
                primitive_data[serialized_name] = None
            else:
                primitive_data[serialized_name] = field.to_primitive(value,
                                                                     context)

        return primitive_data

    def export_loop(self, model_instance, field_converter,
                    role=None, print_none=False):
        """
        Calls the main `export_loop` implementation because they are both
        supposed to operate on models.
        """
        if isinstance(model_instance, self.model_class):
            model_class = model_instance.__class__
        else:
            model_class = self.model_class

        shaped = export_loop(model_class, model_instance,
                             field_converter,
                             role=role, print_none=print_none)

        if shaped and len(shaped) == 0 and self.allow_none():
            return shaped
        elif shaped:
            return shaped
        elif print_none:
            return shaped

class ListType(MultiType):

    MESSAGES = {
        'at_least_item': _(u"Please provide at least {0} item."),
        'more_than_item': _(u"Please provide no more than {0} item."),
    }

    def __init__(self, field, min_size=None, max_size=None, **kwargs):

        if not isinstance(field, BaseType):
            compound_field = kwargs.pop('compound_field', None)
            field = self.init_compound_field(field, compound_field, **kwargs)

        self.field = field
        self.min_size = min_size
        self.max_size = max_size

        validators = [self.check_length, self.validate_items] + kwargs.pop("validators", [])

        super(ListType, self).__init__(validators=validators, **kwargs)

    @property
    def model_class(self):
        return self.field.model_class

    def _force_list(self, value):
        if value is None or value == EMPTY_LIST:
            return []

        try:
            if isinstance(value, basestring):
                raise TypeError()

            if isinstance(value, dict):
                return [value[unicode(k)] for k in sorted(map(int, value.keys()))]

            return list(value)
        except TypeError:
            return [value]

    def to_native(self, value, context=None):
        items = self._force_list(value)

        return [self.field.to_native(item, context) for item in items]

    def check_length(self, value):
        list_length = len(value) if value else 0

        if self.min_size is not None and list_length < self.min_size:
            message = ({
                True: self.messages['at_least_item'],
                False: self.messages['at_least_item'],
            }[self.min_size == 1]).format(self.min_size)
            raise ValidationError(message)

        if self.max_size is not None and list_length > self.max_size:
            message = ({
                True: self.messages['more_than_item'],
                False: self.messages['more_than_item'],
            }[self.max_size == 1]).format(self.max_size)
            raise ValidationError(message)

    def validate_items(self, items):
        errors = []
        for item in items:
            try:
                self.field.validate(item)
            except ValidationError as exc:
                errors.append(exc.messages)
        if errors:
            raise ValidationError(errors)

    def to_primitive(self, value, context=None):
        return [self.field.to_primitive(item, context) for item in value]

    def export_loop(self, list_instance, field_converter,
                    role=None, print_none=False):
        """Loops over each item in the model and applies either the field
        transform or the multitype transform.  Essentially functions the same
        as `transforms.export_loop`.
        """
        data = []
        for value in list_instance:
            if hasattr(self.field, 'export_loop'):
                shaped = self.field.export_loop(value, field_converter,
                                                role=role)
                feels_empty = shaped and len(shaped) == 0
            else:
                shaped = field_converter(self.field, value)
                feels_empty = shaped is None

            # Print if we want empty or found a value
            if feels_empty and self.field.allow_none():
                data.append(shaped)
            elif shaped is not None:
                data.append(shaped)
            elif print_none:
                data.append(shaped)

        # Return data if the list contains anything
        if len(data) > 0:
            return data
        elif len(data) == 0 and self.allow_none():
            return data
        elif print_none:
            return data

class DictType(MultiType):

    MESSAGES = {
        'only_dict': _(u"Only dictionaries may be used in a DictType."),
    }

    def __init__(self, field, coerce_key=None, **kwargs):
        if not isinstance(field, BaseType):
            compound_field = kwargs.pop('compound_field', None)
            field = self.init_compound_field(field, compound_field, **kwargs)

        self.coerce_key = coerce_key or unicode
        self.field = field

        validators = [self.validate_items] + kwargs.pop("validators", [])

        super(DictType, self).__init__(validators=validators, **kwargs)

    @property
    def model_class(self):
        return self.field.model_class

    def to_native(self, value, safe=False, context=None):
        if value == EMPTY_DICT:
            value = {}

        value = value or {}

        if not isinstance(value, dict):
            raise ValidationError(self.messages['only_dict'])

        return dict((self.coerce_key(k), self.field.to_native(v, context))
                    for k, v in iteritems(value))

    def validate_items(self, items):
        errors = {}
        for key, value in iteritems(items):
            try:
                self.field.validate(value)
            except ValidationError as exc:
                errors[key] = exc

        if errors:
            raise ValidationError(errors)

    def to_primitive(self, value, context=None):
        return dict((unicode(k), self.field.to_primitive(v, context))
                    for k, v in iteritems(value))

    def export_loop(self, dict_instance, field_converter,
                    role=None, print_none=False):
        """Loops over each item in the model and applies either the field
        transform or the multitype transform.  Essentially functions the same
        as `transforms.export_loop`.
        """
        data = {}

        for key, value in iteritems(dict_instance):
            if hasattr(self.field, 'export_loop'):
                shaped = self.field.export_loop(value, field_converter,
                                                role=role)
                feels_empty = shaped and len(shaped) == 0
            else:
                shaped = field_converter(self.field, value)
                feels_empty = shaped is None

            if feels_empty and self.field.allow_none():
                data[key] = shaped
            elif shaped is not None:
                data[key] = shaped
            elif print_none:
                data[key] = shaped

        if len(data) > 0:
            return data
        elif len(data) == 0 and self.allow_none():
            return data
        elif print_none:
            return data

class PolyModelType(MultiType):

    def __init__(self, model_classes, **kwargs):

        if isinstance(model_classes, type) and issubclass(model_classes, Model):
            self.model_classes = (model_classes,)
            allow_subclasses = True
        elif isinstance(model_classes, Iterable) \
          and not isinstance(model_classes, basestring):
            self.model_classes = tuple(model_classes)
            allow_subclasses = False
        else:
            raise Exception("The first argument to PolyModelType.__init__() "
                            "must be a model or an iterable.")

        validators = kwargs.pop("validators", [])
        self.strict = kwargs.pop("strict", True)
        self.claim_function = kwargs.pop("claim_function", None)
        self.allow_subclasses = kwargs.pop("allow_subclasses", allow_subclasses)

        def validate_model(model_instance):
            model_instance.validate()
            return model_instance

        MultiType.__init__(self, validators=[validate_model] + validators, **kwargs)

    def __repr__(self):
        return object.__repr__(self)[:-1] + ' for %s>' % str(self.model_classes)

    def is_allowed_model(self, model_instance):
        if self.allow_subclasses:
            if isinstance(model_instance, self.model_classes):
                return True
        else:
            if model_instance.__class__ in self.model_classes:
                return True
        return False

    def to_native(self, value, mapping=None, context=None):

        if mapping is None:
            mapping = {}
        if value is None:
            return None
        if self.is_allowed_model(value):
            return value
        if not isinstance(value, dict):
            if len(self.model_classes) > 1:
                instanceof_msg = 'one of: {}'.format(', '.join(
                    cls.__name__ for cls in self.model_classes))
            else:
                instanceof_msg = self.model_classes[0].__name__
            raise ConversionError(u'Please use a mapping for this field or '
                                    'an instance of {}'.format(instanceof_msg))

        model_class = self.find_model(value)
        model = model_class()
        return model.import_data(value, mapping=mapping, context=context,
                                 strict=self.strict)

    def find_model(self, data):
        """Finds the intended type by consulting potential classes or `claim_function`."""

        chosen_class = None
        if self.claim_function:
            chosen_class = self.claim_function(self, data)
        else:
            candidates = self.model_classes
            if self.allow_subclasses:
                candidates = itertools.chain.from_iterable(
                                 ([m] + m._subclasses for m in candidates))
            fallback = None
            matching_classes = []
            for cls in candidates:
                match = None
                if '_claim_polymorphic' in cls.__dict__:
                    match = cls._claim_polymorphic(data)
                elif not fallback: # The first model that doesn't define the hook
                    fallback = cls # can be used as a default if there's no match.
                if match:
                    matching_classes.append(cls)
            if not matching_classes and fallback:
                chosen_class = fallback
            elif len(matching_classes) == 1:
                chosen_class = matching_classes[0]
            else:
                raise Exception("Got ambiguous input for polymorphic field")
        if chosen_class:
            return chosen_class
        else:
            raise Exception("Input for polymorphic field did not match any model")

    def export_loop(self, model_instance, field_converter,
                    role=None, print_none=False):

        model_class = model_instance.__class__
        if not self.is_allowed_model(model_instance):
            raise Exception("Cannot export: {} is not an allowed type".format(model_class))

        shaped = export_loop(model_class, model_instance,
                             field_converter,
                             role=role, print_none=print_none)

        if shaped and len(shaped) == 0 and self.allow_none():
            return shaped
        elif shaped:
            return shaped
        elif print_none:
            return shaped

<|endoftext|>


<commit_before>"""Tune the parameters of an estimator by cross-validation"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>,
#         Gael Varoquaux    <gael.varoquaux@normalesup.org>
# License: BSD Style.

import copy
import itertools

import numpy as np
import scipy.sparse as sp

from .externals.joblib import Parallel, delayed
from .cross_val import KFold, StratifiedKFold
from .base import BaseEstimator, is_classifier, clone

try:
    from itertools import product
except:
    def product(*args, **kwds):
        pools = map(tuple, args) * kwds.get('repeat', 1)
        result = [[]]
        for pool in pools:
            result = [x+[y] for x in result for y in pool]
        for prod in result:
            yield tuple(prod)

def iter_grid(param_grid):
    """Generators on the combination of the various parameter lists given

    Parameters
    -----------
    kwargs: keyword arguments, lists
        Each keyword argument must be a list of values that should
        be explored.

    Returns
    --------
    params: dictionary
        Dictionnary with the input parameters taking the various
        values succesively.

    Examples
    ---------
    >>> from scikits.learn.grid_search import iter_grid
    >>> param_grid = {'a':[1, 2], 'b':[True, False]}
    >>> list(iter_grid(param_grid))
    [{'a': 1, 'b': True}, {'a': 1, 'b': False}, {'a': 2, 'b': True}, {'a': 2, 'b': False}]

    """
    if hasattr(param_grid, 'has_key'):
        param_grid = [param_grid]
    for p in param_grid:
        # Always sort the keys of a dictionary, for reproducibility
        items = sorted(p.items())
        keys, values = zip(*items)
        for v in product(*values):
            params = dict(zip(keys, v))
            yield params

def fit_grid_point(X, y, base_clf, clf_params, train, test, loss_func,
                score_func, verbose, **fit_params):
    """Run fit on one set of parameters

    Returns the score and the instance of the classifier
    """
    # update parameters of the classifier after a copy of its base structure
    clf = copy.deepcopy(base_clf)
    clf._set_params(**clf_params)

    if isinstance(X, list) or isinstance(X, tuple):
        X_train = [X[i] for i, cond in enumerate(train) if cond]
        X_test = [X[i] for i, cond in enumerate(test) if cond]
    else:
        if sp.issparse(X):
            # For sparse matrices, slicing only works with indices
            # (no masked array). Convert to CSR format for efficiency and
            # because some sparse formats don't support row slicing.
            X = sp.csr_matrix(X)
            ind = np.arange(X.shape[0])
            train = ind[train]
            test = ind[test]
        X_train = X[train]
        X_test = X[test]
    if y is not None:
        y_test  = y[test]
        y_train = y[train]
    else:
        y_test  = None
        y_train = None

    if verbose > 1:
        print '%s\nFitting %s' % (80*'.', clf)
    clf.fit(X_train, y_train, **fit_params)

    if loss_func is not None:
        y_pred = clf.predict(X_test)
        this_score = -loss_func(y_test, y_pred)
    elif score_func is not None:
        y_pred = clf.predict(X_test)
        this_score = score_func(y_test, y_pred)
    else:
        this_score = clf.score(X_test, y_test)

    if y is not None:
        this_n_test_samples = y.shape[0]
    else:
        this_n_test_samples = X.shape[0]

    return this_score, clf, this_n_test_samples

class GridSearchCV(BaseEstimator):
    """Grid search on the parameters of a classifier

    Important members are fit, predict.

    GridSearchCV implements a "fit" method and a "predict" method like
    any classifier except that the parameters of the classifier
    used to predict is optimized by cross-validation

    Parameters
    ----------
    estimator: object type that implements the "fit" and "predict" methods
        A object of that type is instanciated for each grid point

    param_grid: dict
        a dictionary of parameters that are used the generate the grid

    loss_func: callable, optional
        function that takes 2 arguments and compares them in
        order to evaluate the performance of prediciton (small is good)
        if None is passed, the score of the estimator is maximized

    score_func: callable, optional
        function that takes 2 arguments and compares them in
        order to evaluate the performance of prediciton (big is good)
        if None is passed, the score of the estimator is maximized

    fit_params : dict, optional
        parameters to pass to the fit method

    n_jobs: int, optional
        number of jobs to run in parallel (default 1)

    iid: boolean, optional
        If True, the data is assumed to be identically distributed across
        the folds, and the loss minimized is the total loss per sample,
        and not the mean loss across the folds.

    cv : crossvalidation generator
        see scikits.learn.cross_val module

    refit: boolean
        refit the best estimator with the entire dataset

    verbose: integer
        Controls the verbosity: the higher, the more messages.

    Examples
    --------
    >>> from scikits.learn import svm, grid_search, datasets
    >>> iris = datasets.load_iris()
    >>> parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
    >>> svr = svm.SVR()
    >>> clf = grid_search.GridSearchCV(svr, parameters)
    >>> clf.fit(iris.data, iris.target) # doctest: +ELLIPSIS
    GridSearchCV(n_jobs=1, fit_params={}, loss_func=None, refit=True, cv=None,
           iid=True,
           estimator=SVR(kernel='rbf', C=1.0, probability=False, ...
           ...

    Notes
    ------

    The parameters selected are those that maximize the score of the
    left out data, unless an explicit score_func is passed in which
    case it is used instead. If a loss function loss_func is passed,
    it overrides the score functions and is minimized.

    """

    def __init__(self, estimator, param_grid, loss_func=None, score_func=None,
                 fit_params={}, n_jobs=1, iid=True, refit=True, cv=None,
                 verbose=0,
                 ):
        assert hasattr(estimator, 'fit') and (hasattr(estimator, 'predict')
                        or hasattr(estimator, 'score')), (
            "estimator should a be an estimator implementing 'fit' and "
            "'predict' or 'score' methods, %s (type %s) was passed" %
                    (estimator, type(estimator))
            )
        if loss_func is None and score_func is None:
            assert hasattr(estimator, 'score'), ValueError(
                    "If no loss_func is specified, the estimator passed "
                    "should have a 'score' method. The estimator %s "
                    "does not." % estimator
                    )

        self.estimator = estimator
        self.param_grid = param_grid
        self.loss_func = loss_func
        self.score_func = score_func
        self.n_jobs = n_jobs
        self.fit_params = fit_params
        self.iid = iid
        self.refit = refit
        self.cv = cv
        self.verbose = verbose

    def fit(self, X, y=None, **params):
        """Run fit with all sets of parameters

        Returns the best classifier

        Parameters
        ----------

        X: array, [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        y: array, [n_samples] or None
            Target vector relative to X, None for unsupervised problems

        """
        self._set_params(**params)
        estimator = self.estimator
        cv        = self.cv
        if cv is None:
            if hasattr(X, 'shape'):
                n_samples = X.shape[0]
            else:
                # support list of unstructured objects on which feature
                # extraction will be applied later in the tranformer chain
                n_samples = len(X)
            if y is not None and is_classifier(estimator):
                cv = StratifiedKFold(y, k=3)
            else:
                cv = KFold(n_samples, k=3)

        grid = iter_grid(self.param_grid)
        base_clf = clone(self.estimator)
        # XXX: Need to make use of Parallel's new pre_dispatch
        out = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
            delayed(fit_grid_point)(
                X, y, base_clf, clf_params, train, test, self.loss_func,
                self.score_func, self.verbose, **self.fit_params)
                    for train, test in cv for clf_params in grid)

        # Out is a list of triplet: score, estimator, n_test_samples
        n_grid_points = len(list(iter_grid(self.param_grid)))
        n_fits        = len(out)
        n_folds       = n_fits//n_grid_points

        scores = list()
        for grid_start in range(0, n_fits, n_folds):
            n_test_samples = 0
            score = 0
            for this_score, estimator, this_n_test_samples in \
                        itertools.islice(out, grid_start, grid_start+n_folds):
                if self.iid:
                    this_score *= this_n_test_samples
                score += this_score
                n_test_samples += this_n_test_samples
            if self.iid:
                score /= float(n_test_samples)
            scores.append((score, estimator))

        # Note: we do not use max(out) to make ties deterministic even if
        # comparison on estimator instances is not deterministic
        best_score = None
        for score, estimator in scores:
            if best_score is None:
                best_score = score
                best_estimator = estimator
            else:
                if score >= best_score:
                    best_score = score
                    best_estimator = estimator

        self.best_score = best_score

        if self.refit:
            # fit the best estimator using the entire dataset
            best_estimator.fit(X, y, **self.fit_params)

        self.best_estimator = best_estimator
        if hasattr(best_estimator, 'predict'):
            self.predict = best_estimator.predict
        if hasattr(best_estimator, 'score'):
            self.score = best_estimator.score

        # Store the computed scores
        grid = iter_grid(self.param_grid)
        # XXX: the name is too specific, it shouldn't have
        # 'grid' in it. Also, we should be retrieving/storing variance
        self.grid_points_scores_ = dict((tuple(clf_params.items()), score)
                    for clf_params, (score, _) in zip(grid, scores))

        return self

    def score(self, X, y=None):
        # This method is overridden during the fit if the best estimator
        # found has a score function.
        y_predicted = self.predict(X)
        return self.score_func(y, y_predicted)
<commit_msg>ENH: GridSearCV: better verbose<commit_after>"""Tune the parameters of an estimator by cross-validation"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>,
#         Gael Varoquaux    <gael.varoquaux@normalesup.org>
# License: BSD Style.

import copy
import itertools

import numpy as np
import scipy.sparse as sp

from .externals.joblib import Parallel, delayed
from .cross_val import KFold, StratifiedKFold
from .base import BaseEstimator, is_classifier, clone

try:
    from itertools import product
except:
    def product(*args, **kwds):
        pools = map(tuple, args) * kwds.get('repeat', 1)
        result = [[]]
        for pool in pools:
            result = [x+[y] for x in result for y in pool]
        for prod in result:
            yield tuple(prod)

def iter_grid(param_grid):
    """Generators on the combination of the various parameter lists given

    Parameters
    -----------
    kwargs: keyword arguments, lists
        Each keyword argument must be a list of values that should
        be explored.

    Returns
    --------
    params: dictionary
        Dictionnary with the input parameters taking the various
        values succesively.

    Examples
    ---------
    >>> from scikits.learn.grid_search import iter_grid
    >>> param_grid = {'a':[1, 2], 'b':[True, False]}
    >>> list(iter_grid(param_grid))
    [{'a': 1, 'b': True}, {'a': 1, 'b': False}, {'a': 2, 'b': True}, {'a': 2, 'b': False}]

    """
    if hasattr(param_grid, 'has_key'):
        param_grid = [param_grid]
    for p in param_grid:
        # Always sort the keys of a dictionary, for reproducibility
        items = sorted(p.items())
        keys, values = zip(*items)
        for v in product(*values):
            params = dict(zip(keys, v))
            yield params

def fit_grid_point(X, y, base_clf, clf_params, train, test, loss_func,
                score_func, verbose, **fit_params):
    """Run fit on one set of parameters

    Returns the score and the instance of the classifier
    """
    # update parameters of the classifier after a copy of its base structure
    clf = copy.deepcopy(base_clf)
    clf._set_params(**clf_params)

    if isinstance(X, list) or isinstance(X, tuple):
        X_train = [X[i] for i, cond in enumerate(train) if cond]
        X_test = [X[i] for i, cond in enumerate(test) if cond]
    else:
        if sp.issparse(X):
            # For sparse matrices, slicing only works with indices
            # (no masked array). Convert to CSR format for efficiency and
            # because some sparse formats don't support row slicing.
            X = sp.csr_matrix(X)
            ind = np.arange(X.shape[0])
            train = ind[train]
            test = ind[test]
        X_train = X[train]
        X_test = X[test]
    if y is not None:
        y_test  = y[test]
        y_train = y[train]
    else:
        y_test  = None
        y_train = None

    if verbose > 1:
        msg = '[GridSearchCV] %s' % (', '.join('%s=%s' % (k, v)
                                     for k, v in clf_params.iteritems()))
        print "%s %s" % (msg, (79-len(msg))*'.')
    clf.fit(X_train, y_train, **fit_params)

    if loss_func is not None:
        y_pred = clf.predict(X_test)
        this_score = -loss_func(y_test, y_pred)
    elif score_func is not None:
        y_pred = clf.predict(X_test)
        this_score = score_func(y_test, y_pred)
    else:
        this_score = clf.score(X_test, y_test)

    if y is not None:
        this_n_test_samples = y.shape[0]
    else:
        this_n_test_samples = X.shape[0]

    return this_score, clf, this_n_test_samples

class GridSearchCV(BaseEstimator):
    """Grid search on the parameters of a classifier

    Important members are fit, predict.

    GridSearchCV implements a "fit" method and a "predict" method like
    any classifier except that the parameters of the classifier
    used to predict is optimized by cross-validation

    Parameters
    ----------
    estimator: object type that implements the "fit" and "predict" methods
        A object of that type is instanciated for each grid point

    param_grid: dict
        a dictionary of parameters that are used the generate the grid

    loss_func: callable, optional
        function that takes 2 arguments and compares them in
        order to evaluate the performance of prediciton (small is good)
        if None is passed, the score of the estimator is maximized

    score_func: callable, optional
        function that takes 2 arguments and compares them in
        order to evaluate the performance of prediciton (big is good)
        if None is passed, the score of the estimator is maximized

    fit_params : dict, optional
        parameters to pass to the fit method

    n_jobs: int, optional
        number of jobs to run in parallel (default 1)

    iid: boolean, optional
        If True, the data is assumed to be identically distributed across
        the folds, and the loss minimized is the total loss per sample,
        and not the mean loss across the folds.

    cv : crossvalidation generator
        see scikits.learn.cross_val module

    refit: boolean
        refit the best estimator with the entire dataset

    verbose: integer
        Controls the verbosity: the higher, the more messages.

    Examples
    --------
    >>> from scikits.learn import svm, grid_search, datasets
    >>> iris = datasets.load_iris()
    >>> parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
    >>> svr = svm.SVR()
    >>> clf = grid_search.GridSearchCV(svr, parameters)
    >>> clf.fit(iris.data, iris.target) # doctest: +ELLIPSIS
    GridSearchCV(n_jobs=1, fit_params={}, loss_func=None, refit=True, cv=None,
           iid=True,
           estimator=SVR(kernel='rbf', C=1.0, probability=False, ...
           ...

    Notes
    ------

    The parameters selected are those that maximize the score of the
    left out data, unless an explicit score_func is passed in which
    case it is used instead. If a loss function loss_func is passed,
    it overrides the score functions and is minimized.

    """

    def __init__(self, estimator, param_grid, loss_func=None, score_func=None,
                 fit_params={}, n_jobs=1, iid=True, refit=True, cv=None,
                 verbose=0,
                 ):
        assert hasattr(estimator, 'fit') and (hasattr(estimator, 'predict')
                        or hasattr(estimator, 'score')), (
            "estimator should a be an estimator implementing 'fit' and "
            "'predict' or 'score' methods, %s (type %s) was passed" %
                    (estimator, type(estimator))
            )
        if loss_func is None and score_func is None:
            assert hasattr(estimator, 'score'), ValueError(
                    "If no loss_func is specified, the estimator passed "
                    "should have a 'score' method. The estimator %s "
                    "does not." % estimator
                    )

        self.estimator = estimator
        self.param_grid = param_grid
        self.loss_func = loss_func
        self.score_func = score_func
        self.n_jobs = n_jobs
        self.fit_params = fit_params
        self.iid = iid
        self.refit = refit
        self.cv = cv
        self.verbose = verbose

    def fit(self, X, y=None, **params):
        """Run fit with all sets of parameters

        Returns the best classifier

        Parameters
        ----------

        X: array, [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        y: array, [n_samples] or None
            Target vector relative to X, None for unsupervised problems

        """
        self._set_params(**params)
        estimator = self.estimator
        cv        = self.cv
        if cv is None:
            if hasattr(X, 'shape'):
                n_samples = X.shape[0]
            else:
                # support list of unstructured objects on which feature
                # extraction will be applied later in the tranformer chain
                n_samples = len(X)
            if y is not None and is_classifier(estimator):
                cv = StratifiedKFold(y, k=3)
            else:
                cv = KFold(n_samples, k=3)

        grid = iter_grid(self.param_grid)
        base_clf = clone(self.estimator)
        # XXX: Need to make use of Parallel's new pre_dispatch
        out = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
            delayed(fit_grid_point)(
                X, y, base_clf, clf_params, train, test, self.loss_func,
                self.score_func, self.verbose, **self.fit_params)
                    for train, test in cv for clf_params in grid)

        # Out is a list of triplet: score, estimator, n_test_samples
        n_grid_points = len(list(iter_grid(self.param_grid)))
        n_fits        = len(out)
        n_folds       = n_fits//n_grid_points

        scores = list()
        for grid_start in range(0, n_fits, n_folds):
            n_test_samples = 0
            score = 0
            for this_score, estimator, this_n_test_samples in \
                        itertools.islice(out, grid_start, grid_start+n_folds):
                if self.iid:
                    this_score *= this_n_test_samples
                score += this_score
                n_test_samples += this_n_test_samples
            if self.iid:
                score /= float(n_test_samples)
            scores.append((score, estimator))

        # Note: we do not use max(out) to make ties deterministic even if
        # comparison on estimator instances is not deterministic
        best_score = None
        for score, estimator in scores:
            if best_score is None:
                best_score = score
                best_estimator = estimator
            else:
                if score >= best_score:
                    best_score = score
                    best_estimator = estimator

        self.best_score = best_score

        if self.refit:
            # fit the best estimator using the entire dataset
            best_estimator.fit(X, y, **self.fit_params)

        self.best_estimator = best_estimator
        if hasattr(best_estimator, 'predict'):
            self.predict = best_estimator.predict
        if hasattr(best_estimator, 'score'):
            self.score = best_estimator.score

        # Store the computed scores
        grid = iter_grid(self.param_grid)
        # XXX: the name is too specific, it shouldn't have
        # 'grid' in it. Also, we should be retrieving/storing variance
        self.grid_points_scores_ = dict((tuple(clf_params.items()), score)
                    for clf_params, (score, _) in zip(grid, scores))

        return self

    def score(self, X, y=None):
        # This method is overridden during the fit if the best estimator
        # found has a score function.
        y_predicted = self.predict(X)
        return self.score_func(y, y_predicted)
<|endoftext|>


<commit_before>from models import Framework
from utils import get_json_from_request, json_has_required_keys, \
    json_has_matching_id
from validation import get_validation_errors

def validate_and_return_draft_request(draft_id=0):
    json_payload = get_json_from_request()
    json_has_required_keys(json_payload, ['services'])
    if draft_id:
        json_has_matching_id(json_payload['services'], draft_id)
    return json_payload['services']

def get_draft_validation_errors(draft_json, lot,
                                framework_id=0, slug=None, required=None):
    if not slug and not framework_id:
        raise Exception('Validation requires either framework_id or slug')
    if not slug:
        # TODO: Get framework slug from Framework table once it exists.
        # framework = Framework.query.filter(
        #     Framework.id == framework_id
        # ).first()
        # slug = framework.slug
        if framework_id == 1:
            slug = "g-cloud-6"
        else:
            slug = "g-cloud-7"
    errs = get_validation_errors(
        "services-{0}-{1}".format(slug, lot.lower()),
        draft_json,
        enforce_required=False,
        required_fields=required
    )
    return errs
<commit_msg>Relative imports for Python 3<commit_after>from .models import Framework
from .utils import get_json_from_request, json_has_required_keys, \
    json_has_matching_id
from .validation import get_validation_errors

def validate_and_return_draft_request(draft_id=0):
    json_payload = get_json_from_request()
    json_has_required_keys(json_payload, ['services'])
    if draft_id:
        json_has_matching_id(json_payload['services'], draft_id)
    return json_payload['services']

def get_draft_validation_errors(draft_json, lot,
                                framework_id=0, slug=None, required=None):
    if not slug and not framework_id:
        raise Exception('Validation requires either framework_id or slug')
    if not slug:
        # TODO: Get framework slug from Framework table once it exists.
        # framework = Framework.query.filter(
        #     Framework.id == framework_id
        # ).first()
        # slug = framework.slug
        if framework_id == 1:
            slug = "g-cloud-6"
        else:
            slug = "g-cloud-7"
    errs = get_validation_errors(
        "services-{0}-{1}".format(slug, lot.lower()),
        draft_json,
        enforce_required=False,
        required_fields=required
    )
    return errs
<|endoftext|>


<commit_before>from __future__ import annotations
from typing import (
    Any, Callable, Iterable, Optional, Tuple, TYPE_CHECKING, Union
)

import numpy as np
from scipy.optimize import OptimizeResult
from ._constraints import old_bound_to_new, Bounds
from ._direct import direct as _direct  # type: ignore

if TYPE_CHECKING:
    import numpy.typing as npt
    from _typeshed import NoneType

__all__ = ['direct']

ERROR_MESSAGES = (
    "Number of function evaluations done is larger than maxfun={}",
    "Number of iterations is larger than maxiter={}",
    "u[i] < l[i] for some i",
    "maxfun is too large",
    "Initialization failed",
    "There was an error in the creation of the sample points",
    "An error occured while the function was sampled",
    "Maximum number of levels has been reached.",
    "Forced stop",
    "Invalid arguments",
    "Out of memory",
)

SUCCESS_MESSAGES = (
    ("The best function value found is within a relative error={} "
     "of the (known) global optimum f_min"),
    ("The volume of the hyperrectangle containing the lowest function value "
     "found is below vol_tol={}"),
    ("The side length measure of the hyperrectangle containing the lowest "
     "function value found is below len_tol={}"),
)

def direct(
    func: Callable[[npt.ArrayLike, Tuple[Any]], float],
    bounds: Union[Iterable, Bounds],
    *,
    args: tuple = (),
    eps: float = 1e-4,
    maxfun: Union[int, None] = None,
    maxiter: int = 1000,
    locally_biased: bool = True,
    f_min: float = -np.inf,
    f_min_rtol: float = 1e-4,
    vol_tol: float = 1e-16,
    len_tol: float = 1e-6,
    callback: Optional[Callable[[npt.ArrayLike], NoneType]] = None
) -> OptimizeResult:
    """
    Finds the global minimum of a function using the
    DIRECT algorithm.

    Parameters
    ----------
    func : callable
        The objective function to be minimized.
        ``func(x, *args) -> float``
        where ``x`` is an 1-D array with shape (n,) and ``args`` is a tuple of
        the fixed parameters needed to completely specify the function.
    bounds : sequence or `Bounds`
        Bounds for variables. There are two ways to specify the bounds:

        1. Instance of `Bounds` class.
        2. ``(min, max)`` pairs for each element in ``x``.

    args : tuple, optional
        Any additional fixed parameters needed to
        completely specify the objective function.
    eps : float, optional
        Minimal required difference of the objective function values
        between the current best hyperrectangle and the next potentially
        optimal hyperrectangle to be divided. In consequence, `eps` serves as a
        tradeoff between local and global search: the smaller, the more local
        the search becomes. Default is 1e-4.
    maxfun : int or None, optional
        Approximate upper bound on objective function evaluations.
        If `None`, will be automatically set to ``1000 * N`` where ``N``
        represents the number of dimensions. Will be capped if necessary to
        limit DIRECT's RAM usage to app. 1GiB. This will only occur for very
        high dimensional problems and excessive `max_fun`. Default is `None`.
    maxiter : int, optional
        Maximum number of iterations. Default is 1000.
    locally_biased : bool, optional
        If `True` (default), use the locally biased variant of the
        algorithm known as DIRECT_L. If `False`, use the original unbiased
        DIRECT algorithm. For hard problems with many local minima,
        `False` is recommended.
    f_min : float, optional
        Function value of the global optimum. Set this value only if the
        global optimum is known. Default is ``-np.inf``, so that this
        termination criterion is deactivated.
    f_min_rtol : float, optional
        Terminate the optimization once the relative error between the
        current best minimum `f` and the supplied global minimum `f_min`
        is smaller than `f_min_rtol`. This parameter is only used if
        `f_min` is also set. Default is 1e-4.
    vol_tol : float, optional
        Terminate the optimization once the volume of the hyperrectangle
        containing the lowest function value is smaller than `vol_tol`
        of the complete search space. Must lie between 0 and 1.
        Default is 1e-16.
    len_tol : float, optional
        If `locally_biased=True`, terminate the optimization once half of
        the normalized maximal side length of the hyperrectangle containing
        the lowest function value is smaller than `len_tol`.
        If `locally_biased=False`, terminate the optimization once half of
        the normalized diagonal of the hyperrectangle containing the lowest
        function value is smaller than `len_tol`. Must lie between 0 and 1.
        Default is 1e-6.
    callback : callable, optional
        A callback function with signature ``callback(xk)`` where ``xk``
        represents the best function value found so far.

    Returns
    -------
    res : OptimizeResult
        The optimization result represented as a ``OptimizeResult`` object.
        Important attributes are: ``x`` the solution array, ``success`` a
        Boolean flag indicating if the optimizer exited successfully and
        ``message`` which describes the cause of the termination. See
        `OptimizeResult` for a description of other attributes.

    Notes
    -----
    DIviding RECTangles (DIRECT) is a deterministic global
    optimization algorithm capable of minimizing a black box function with
    its variables subject to lower and upper bound constraints by sampling
    potential solutions in the search space [1]_. The algorithm starts by
    normalising the search space to an n-dimensional unit hypercube.
    It samples the function at the center of this hypercube and at 2n
    (n is the number of variables) more points, 2 in each coordinate
    direction. Using these function values, DIRECT then divides the
    domain into hyperrectangles, each having exactly one of the sampling
    points as its center. In each iteration, DIRECT chooses, using the `eps`
    parameter which defaults to 1e-4, some of the existing hyperrectangles
    to be further divided. This division process continues until either the
    maximum number of iterations or maximum function evaluations allowed
    are exceeded, or the hyperrectangle containing the minimal value found
    so far becomes small enough. If `f_min` is specified, the optimization
    will stop once this function value is reached within a relative tolerance.
    The locally biased variant of DIRECT (originally called DIRECT_L) [2]_ is
    used by default. It makes the search more locally biased and more
    efficient for cases with only a few local minima.

    A note about termination criteria: `vol_tol` refers to the volume of the
    hyperrectangle containing the lowest function value found so far. This
    volume decreases exponentially with increasing dimensionality of the
    problem. Therefore `vol_tol` should be decreased to avoid premature
    termination of the algorithm for higher dimensions. This does not hold
    for `len_tol`: it refers either to half of the maximal side length
    (for ``locally_biased=True``) or half of the diagonal of the
    hyperrectangle (for ``locally_biased=False``).

    This code is based on the DIRECT 2.0.4 Fortran code by Gablonsky et al. at
    https://ctk.math.ncsu.edu/SOFTWARE/DIRECTv204.tar.gz .
    This original version was initially converted via f2c and then cleaned up
    and reorganized by Steven G. Johnson, August 2007, for the NLopt project.
    The `direct` function wraps the C implementation.

    .. versionadded:: 1.9.0

    References
    ----------
    .. [1] Jones, D.R., Perttunen, C.D. & Stuckman, B.E. Lipschitzian
        optimization without the Lipschitz constant. J Optim Theory Appl
        79, 157-181 (1993).
    .. [2] Gablonsky, J., Kelley, C. A Locally-Biased form of the DIRECT
        Algorithm. Journal of Global Optimization 21, 27-37 (2001).

    Examples
    --------
    The following example is a 2-D problem with four local minima: minimizing
    the Styblinski-Tang function
    (https://en.wikipedia.org/wiki/Test_functions_for_optimization).

    >>> from scipy.optimize import direct, Bounds
    >>> def styblinski_tang(pos):
    ...     x, y = pos
    ...     return 0.5 * (x**4 - 16*x**2 + 5*x + y**4 - 16*y**2 + 5*y)
    >>> bounds = Bounds([-4., -4.], [4., 4.])
    >>> result = direct(styblinski_tang, bounds)
    >>> result.x, result.fun, result.nfev
    array([-2.90321597, -2.90321597]), -78.3323279095383, 2011

    The correct global minimum was found but with a huge number of function
    evaluations (2011). Loosening the termination tolerances `vol_tol` and
    `len_tol` can be used to stop DIRECT earlier.

    >>> result = direct(styblinski_tang, bounds, len_tol=1e-3)
    >>> result.x, result.fun, result.nfev
    array([-2.9044353, -2.9044353]), -78.33230330754142, 207

    """
    # convert bounds to new Bounds class if necessary
    if not isinstance(bounds, Bounds):
        if isinstance(bounds, list) or isinstance(bounds, tuple):
            lb, ub = old_bound_to_new(bounds)
            bounds = Bounds(lb, ub)
        else:
            message = ("bounds must be a sequence or "
                       "instance of Bounds class")
            raise ValueError(message)

    lb = np.ascontiguousarray(bounds.lb, dtype=np.float64)
    ub = np.ascontiguousarray(bounds.ub, dtype=np.float64)

    # validate bounds
    if not np.all(lb < ub):
        raise ValueError('Bounds are not consistent min < max')
    if not len(lb) == len(ub):
        raise ValueError('Bounds do not have the same dimensions')

    # check for infs and nans
    if (np.any(np.isinf(lb)) or np.any(np.isinf(ub))):
        raise ValueError("Bounds must not be inf.")
    if (np.any(np.isnan(lb)) or np.any(np.isnan(ub))):
        raise ValueError("Bounds must not be NaN.")

    # validate tolerances
    if (vol_tol < 0 or vol_tol > 1):
        raise ValueError("vol_tol must be between 0 and 1.")
    if (len_tol < 0 or len_tol > 1):
        raise ValueError("len_tol must be between 0 and 1.")
    if (f_min_rtol < 0 or f_min_rtol > 1):
        raise ValueError("f_min_rtol must be between 0 and 1.")

    # validate maxfun and maxiter
    if maxfun is None:
        maxfun = 1000 * lb.shape[0]
    if not isinstance(maxfun, int):
        raise ValueError("maxfun must be of type int.")
    if maxfun < 0:
        raise ValueError("maxfun must be > 0.")
    if not isinstance(maxiter, int):
        raise ValueError("maxiter must be of type int.")
    if maxiter < 0:
        raise ValueError("maxiter must be > 0.")

    # validate boolean parameters
    if not isinstance(locally_biased, bool):
        raise ValueError("locally_biased must be True or False.")

    def _func_wrap(x, args=None):
        x = np.asarray(x)
        if args is None:
            f = func(x)
        else:
            f = func(x, *args)
        return f

    # TODO: fix disp argument
    x, fun, ret_code, nfev, nit = _direct(
        _func_wrap,
        np.asarray(lb), np.asarray(ub),
        args,
        False, eps, maxfun, maxiter,
        locally_biased,
        f_min, f_min_rtol,
        vol_tol, len_tol, callback
    )

    format_val = (maxfun, maxiter, f_min_rtol, vol_tol, len_tol)
    if ret_code > 2:
        message = SUCCESS_MESSAGES[ret_code - 3].format(
                    format_val[ret_code - 1])
    elif 0 < ret_code <= 2:
        message = ERROR_MESSAGES[ret_code - 1].format(format_val[ret_code - 1])
    elif 0 > ret_code > -100:
        message = ERROR_MESSAGES[abs(ret_code) + 1]
    else:
        message = ERROR_MESSAGES[ret_code + 99]

    return OptimizeResult(x=np.asarray(x), fun=fun, status=ret_code,
                          success=ret_code > 2, message=message,
                          nfev=nfev, nit=nit)
<commit_msg>MAINT: optimize: always return a float from goal functional wrapper (#16640)<commit_after>from __future__ import annotations
from typing import (
    Any, Callable, Iterable, Optional, Tuple, TYPE_CHECKING, Union
)

import numpy as np
from scipy.optimize import OptimizeResult
from ._constraints import old_bound_to_new, Bounds
from ._direct import direct as _direct  # type: ignore

if TYPE_CHECKING:
    import numpy.typing as npt
    from _typeshed import NoneType

__all__ = ['direct']

ERROR_MESSAGES = (
    "Number of function evaluations done is larger than maxfun={}",
    "Number of iterations is larger than maxiter={}",
    "u[i] < l[i] for some i",
    "maxfun is too large",
    "Initialization failed",
    "There was an error in the creation of the sample points",
    "An error occured while the function was sampled",
    "Maximum number of levels has been reached.",
    "Forced stop",
    "Invalid arguments",
    "Out of memory",
)

SUCCESS_MESSAGES = (
    ("The best function value found is within a relative error={} "
     "of the (known) global optimum f_min"),
    ("The volume of the hyperrectangle containing the lowest function value "
     "found is below vol_tol={}"),
    ("The side length measure of the hyperrectangle containing the lowest "
     "function value found is below len_tol={}"),
)

def direct(
    func: Callable[[npt.ArrayLike, Tuple[Any]], float],
    bounds: Union[Iterable, Bounds],
    *,
    args: tuple = (),
    eps: float = 1e-4,
    maxfun: Union[int, None] = None,
    maxiter: int = 1000,
    locally_biased: bool = True,
    f_min: float = -np.inf,
    f_min_rtol: float = 1e-4,
    vol_tol: float = 1e-16,
    len_tol: float = 1e-6,
    callback: Optional[Callable[[npt.ArrayLike], NoneType]] = None
) -> OptimizeResult:
    """
    Finds the global minimum of a function using the
    DIRECT algorithm.

    Parameters
    ----------
    func : callable
        The objective function to be minimized.
        ``func(x, *args) -> float``
        where ``x`` is an 1-D array with shape (n,) and ``args`` is a tuple of
        the fixed parameters needed to completely specify the function.
    bounds : sequence or `Bounds`
        Bounds for variables. There are two ways to specify the bounds:

        1. Instance of `Bounds` class.
        2. ``(min, max)`` pairs for each element in ``x``.

    args : tuple, optional
        Any additional fixed parameters needed to
        completely specify the objective function.
    eps : float, optional
        Minimal required difference of the objective function values
        between the current best hyperrectangle and the next potentially
        optimal hyperrectangle to be divided. In consequence, `eps` serves as a
        tradeoff between local and global search: the smaller, the more local
        the search becomes. Default is 1e-4.
    maxfun : int or None, optional
        Approximate upper bound on objective function evaluations.
        If `None`, will be automatically set to ``1000 * N`` where ``N``
        represents the number of dimensions. Will be capped if necessary to
        limit DIRECT's RAM usage to app. 1GiB. This will only occur for very
        high dimensional problems and excessive `max_fun`. Default is `None`.
    maxiter : int, optional
        Maximum number of iterations. Default is 1000.
    locally_biased : bool, optional
        If `True` (default), use the locally biased variant of the
        algorithm known as DIRECT_L. If `False`, use the original unbiased
        DIRECT algorithm. For hard problems with many local minima,
        `False` is recommended.
    f_min : float, optional
        Function value of the global optimum. Set this value only if the
        global optimum is known. Default is ``-np.inf``, so that this
        termination criterion is deactivated.
    f_min_rtol : float, optional
        Terminate the optimization once the relative error between the
        current best minimum `f` and the supplied global minimum `f_min`
        is smaller than `f_min_rtol`. This parameter is only used if
        `f_min` is also set. Default is 1e-4.
    vol_tol : float, optional
        Terminate the optimization once the volume of the hyperrectangle
        containing the lowest function value is smaller than `vol_tol`
        of the complete search space. Must lie between 0 and 1.
        Default is 1e-16.
    len_tol : float, optional
        If `locally_biased=True`, terminate the optimization once half of
        the normalized maximal side length of the hyperrectangle containing
        the lowest function value is smaller than `len_tol`.
        If `locally_biased=False`, terminate the optimization once half of
        the normalized diagonal of the hyperrectangle containing the lowest
        function value is smaller than `len_tol`. Must lie between 0 and 1.
        Default is 1e-6.
    callback : callable, optional
        A callback function with signature ``callback(xk)`` where ``xk``
        represents the best function value found so far.

    Returns
    -------
    res : OptimizeResult
        The optimization result represented as a ``OptimizeResult`` object.
        Important attributes are: ``x`` the solution array, ``success`` a
        Boolean flag indicating if the optimizer exited successfully and
        ``message`` which describes the cause of the termination. See
        `OptimizeResult` for a description of other attributes.

    Notes
    -----
    DIviding RECTangles (DIRECT) is a deterministic global
    optimization algorithm capable of minimizing a black box function with
    its variables subject to lower and upper bound constraints by sampling
    potential solutions in the search space [1]_. The algorithm starts by
    normalising the search space to an n-dimensional unit hypercube.
    It samples the function at the center of this hypercube and at 2n
    (n is the number of variables) more points, 2 in each coordinate
    direction. Using these function values, DIRECT then divides the
    domain into hyperrectangles, each having exactly one of the sampling
    points as its center. In each iteration, DIRECT chooses, using the `eps`
    parameter which defaults to 1e-4, some of the existing hyperrectangles
    to be further divided. This division process continues until either the
    maximum number of iterations or maximum function evaluations allowed
    are exceeded, or the hyperrectangle containing the minimal value found
    so far becomes small enough. If `f_min` is specified, the optimization
    will stop once this function value is reached within a relative tolerance.
    The locally biased variant of DIRECT (originally called DIRECT_L) [2]_ is
    used by default. It makes the search more locally biased and more
    efficient for cases with only a few local minima.

    A note about termination criteria: `vol_tol` refers to the volume of the
    hyperrectangle containing the lowest function value found so far. This
    volume decreases exponentially with increasing dimensionality of the
    problem. Therefore `vol_tol` should be decreased to avoid premature
    termination of the algorithm for higher dimensions. This does not hold
    for `len_tol`: it refers either to half of the maximal side length
    (for ``locally_biased=True``) or half of the diagonal of the
    hyperrectangle (for ``locally_biased=False``).

    This code is based on the DIRECT 2.0.4 Fortran code by Gablonsky et al. at
    https://ctk.math.ncsu.edu/SOFTWARE/DIRECTv204.tar.gz .
    This original version was initially converted via f2c and then cleaned up
    and reorganized by Steven G. Johnson, August 2007, for the NLopt project.
    The `direct` function wraps the C implementation.

    .. versionadded:: 1.9.0

    References
    ----------
    .. [1] Jones, D.R., Perttunen, C.D. & Stuckman, B.E. Lipschitzian
        optimization without the Lipschitz constant. J Optim Theory Appl
        79, 157-181 (1993).
    .. [2] Gablonsky, J., Kelley, C. A Locally-Biased form of the DIRECT
        Algorithm. Journal of Global Optimization 21, 27-37 (2001).

    Examples
    --------
    The following example is a 2-D problem with four local minima: minimizing
    the Styblinski-Tang function
    (https://en.wikipedia.org/wiki/Test_functions_for_optimization).

    >>> from scipy.optimize import direct, Bounds
    >>> def styblinski_tang(pos):
    ...     x, y = pos
    ...     return 0.5 * (x**4 - 16*x**2 + 5*x + y**4 - 16*y**2 + 5*y)
    >>> bounds = Bounds([-4., -4.], [4., 4.])
    >>> result = direct(styblinski_tang, bounds)
    >>> result.x, result.fun, result.nfev
    array([-2.90321597, -2.90321597]), -78.3323279095383, 2011

    The correct global minimum was found but with a huge number of function
    evaluations (2011). Loosening the termination tolerances `vol_tol` and
    `len_tol` can be used to stop DIRECT earlier.

    >>> result = direct(styblinski_tang, bounds, len_tol=1e-3)
    >>> result.x, result.fun, result.nfev
    array([-2.9044353, -2.9044353]), -78.33230330754142, 207

    """
    # convert bounds to new Bounds class if necessary
    if not isinstance(bounds, Bounds):
        if isinstance(bounds, list) or isinstance(bounds, tuple):
            lb, ub = old_bound_to_new(bounds)
            bounds = Bounds(lb, ub)
        else:
            message = ("bounds must be a sequence or "
                       "instance of Bounds class")
            raise ValueError(message)

    lb = np.ascontiguousarray(bounds.lb, dtype=np.float64)
    ub = np.ascontiguousarray(bounds.ub, dtype=np.float64)

    # validate bounds
    if not np.all(lb < ub):
        raise ValueError('Bounds are not consistent min < max')
    if not len(lb) == len(ub):
        raise ValueError('Bounds do not have the same dimensions')

    # check for infs and nans
    if (np.any(np.isinf(lb)) or np.any(np.isinf(ub))):
        raise ValueError("Bounds must not be inf.")
    if (np.any(np.isnan(lb)) or np.any(np.isnan(ub))):
        raise ValueError("Bounds must not be NaN.")

    # validate tolerances
    if (vol_tol < 0 or vol_tol > 1):
        raise ValueError("vol_tol must be between 0 and 1.")
    if (len_tol < 0 or len_tol > 1):
        raise ValueError("len_tol must be between 0 and 1.")
    if (f_min_rtol < 0 or f_min_rtol > 1):
        raise ValueError("f_min_rtol must be between 0 and 1.")

    # validate maxfun and maxiter
    if maxfun is None:
        maxfun = 1000 * lb.shape[0]
    if not isinstance(maxfun, int):
        raise ValueError("maxfun must be of type int.")
    if maxfun < 0:
        raise ValueError("maxfun must be > 0.")
    if not isinstance(maxiter, int):
        raise ValueError("maxiter must be of type int.")
    if maxiter < 0:
        raise ValueError("maxiter must be > 0.")

    # validate boolean parameters
    if not isinstance(locally_biased, bool):
        raise ValueError("locally_biased must be True or False.")

    def _func_wrap(x, args=None):
        x = np.asarray(x)
        if args is None:
            f = func(x)
        else:
            f = func(x, *args)
        # always return a float
        return np.asarray(f).item()

    # TODO: fix disp argument
    x, fun, ret_code, nfev, nit = _direct(
        _func_wrap,
        np.asarray(lb), np.asarray(ub),
        args,
        False, eps, maxfun, maxiter,
        locally_biased,
        f_min, f_min_rtol,
        vol_tol, len_tol, callback
    )

    format_val = (maxfun, maxiter, f_min_rtol, vol_tol, len_tol)
    if ret_code > 2:
        message = SUCCESS_MESSAGES[ret_code - 3].format(
                    format_val[ret_code - 1])
    elif 0 < ret_code <= 2:
        message = ERROR_MESSAGES[ret_code - 1].format(format_val[ret_code - 1])
    elif 0 > ret_code > -100:
        message = ERROR_MESSAGES[abs(ret_code) + 1]
    else:
        message = ERROR_MESSAGES[ret_code + 99]

    return OptimizeResult(x=np.asarray(x), fun=fun, status=ret_code,
                          success=ret_code > 2, message=message,
                          nfev=nfev, nit=nit)
<|endoftext|>


<commit_before>import pprint as pp
import traceback

from django.core.exceptions import ObjectDoesNotExist
from django.db import IntegrityError

from source_data.models import *

def map_indicators(indicator_strings,document_id):

    indicator_mapping = {}

    distinct_indicator_strings = list(set(indicator_strings))

    for indicator_string in distinct_indicator_strings:

        try:
            source_indicator = SourceIndicator.objects.get_or_create(
                indicator_string = indicator_string,
                document_id = document_id,
            )
        except IntegrityError:

            source_indicator = SourceIndicator.objects.get(
                indicator_string = indicator_string)

        try:
            indicator_id = IndicatorMap.objects.get(source_indicator_id = \
                source_indicator.id).master_indicator_id

            indicator_mapping[indicator_string] = indicator_id

    except ObjectDoesNotExist:
            pass
        except AttributeError:
            pass

    return indicator_mapping

def map_campaigns(campaign_strings,document_id):

    campaign_mapping = {}

    distinct_campaign_strings = list(set(campaign_strings))

    for campaign in distinct_campaign_strings:

        try:
            source_campaign = SourceCampaign.objects.create(
                campaign_string = campaign,
                document_id = document_id,
            )
        except IntegrityError:
            source_campaign = SourceCampaign.objects.get(
                campaign_string = campaign)

        try:
            campaign_id = CampaignMap.objects.get(source_campaign_id = \
                source_campaign.id).master_campaign_id

            campaign_mapping[str(campaign)] = campaign_id
        except ObjectDoesNotExist:
            pass
        except AttributeError:
            pass

    return campaign_mapping

def map_regions(region_strings,document_id):

    region_mapping = {}

    distinct_region_strings =  list(set(region_strings))

    for region_string in distinct_region_strings:

        try:
            source_region = SourceRegion.objects.create(\
                region_string=region_string,
                document_id = document_id)
        except IntegrityError:
            source_region = SourceRegion.objects.get(region_string=region_string)

        try:
            region_id = RegionMap.objects.get(source_region_id = \
                source_region.id).master_region_id

            region_mapping[region_string] = region_id
        except ObjectDoesNotExist:
            pass
        except AttributeError:
            pass

    return region_mapping
<commit_msg>fix indentation error<commit_after>import pprint as pp
import traceback

from django.core.exceptions import ObjectDoesNotExist
from django.db import IntegrityError

from source_data.models import *

def map_indicators(indicator_strings,document_id):

    indicator_mapping = {}

    distinct_indicator_strings = list(set(indicator_strings))

    for indicator_string in distinct_indicator_strings:

        try:
            source_indicator = SourceIndicator.objects.get_or_create(
                indicator_string = indicator_string,
                document_id = document_id,
            )
        except IntegrityError:

            source_indicator = SourceIndicator.objects.get(
                indicator_string = indicator_string)

        try:
            indicator_id = IndicatorMap.objects.get(source_indicator_id = \
                source_indicator.id).master_indicator_id

            indicator_mapping[indicator_string] = indicator_id

        except ObjectDoesNotExist:
            pass
        except AttributeError:
            pass

    return indicator_mapping

def map_campaigns(campaign_strings,document_id):

    campaign_mapping = {}

    distinct_campaign_strings = list(set(campaign_strings))

    for campaign in distinct_campaign_strings:

        try:
            source_campaign = SourceCampaign.objects.create(
                campaign_string = campaign,
                document_id = document_id,
            )
        except IntegrityError:
            source_campaign = SourceCampaign.objects.get(
                campaign_string = campaign)

        try:
            campaign_id = CampaignMap.objects.get(source_campaign_id = \
                source_campaign.id).master_campaign_id

            campaign_mapping[str(campaign)] = campaign_id
        except ObjectDoesNotExist:
            pass
        except AttributeError:
            pass

    return campaign_mapping

def map_regions(region_strings,document_id):

    region_mapping = {}

    distinct_region_strings =  list(set(region_strings))

    for region_string in distinct_region_strings:

        try:
            source_region = SourceRegion.objects.create(\
                region_string=region_string,
                document_id = document_id)
        except IntegrityError:
            source_region = SourceRegion.objects.get(region_string=region_string)

        try:
            region_id = RegionMap.objects.get(source_region_id = \
                source_region.id).master_region_id

            region_mapping[region_string] = region_id
        except ObjectDoesNotExist:
            pass
        except AttributeError:
            pass

    return region_mapping
<|endoftext|>


<commit_before>import json
from globalvars import GlobalVars
import parsing

class Post:
    _body = ""
    _body_is_summary = False
    _is_answer = False
    _owner_rep = 1
    _post_id = ""
    _post_score = 0
    _post_site = ""
    _title = ""
    _user_name = ""
    _user_url = ""
    _votes = {'downvotes': None, 'upvotes': None}

    def __init__(self, json_data=None, api_response=None):
        if json_data is not None:
            self._parse_json_post(json_data)
        elif api_response is not None:
            self._parse_api_post(api_response)
        else:
            raise ValueError("Must provide either JSON Data or an API Response object for Post object.")
        # self._title = title
        # self._body = body
        # self._user_name = user_name
        # self._user_url = user_url
        # self._post_site = post_site
        # self._post_id = post_id
        # self._is_answer = is_answer
        # self._body_is_summary = body_is_summary
        # self._owner_rep = owner_rep
        # self._post_score = post_score

    def __repr__(self):
        type_name = type(self).__name__
        dataset = ['title=' + self.title, 'body=' + self.body, 'user_name=' + self.user_name,
                   'user_url=' + self.user_url, 'post_site=' + self.post_site, 'post_id=' + self.post_id,
                   'is_answer=' + str(self.is_answer), 'body_is_summary=' + str(self.body_is_summary),
                   'owner_rep=' + str(self.owner_rep), 'post_score=' + self.post_score]
        return "%s(%s)" % (type_name, ', '.join(dataset))

    def _parse_json_post(self, json_data):
        text_data = json.loads(json_data)["data"]
        if text_data == "hb":
            return

        try:
            data = json.loads(text_data)
        except ValueError:
            GlobalVars.charcoal_hq.send_message(u"Encountered ValueError parsing the following:\n{0}".format(json_data),
                                                False)
            return
        if "ownerUrl" not in data:
            # owner's account doesn't exist anymore, no need to post it in chat:
            # http://chat.stackexchange.com/transcript/message/18380776#18380776
            return
        self._title = data["titleEncodedFancy"]
        self._title = parsing.unescape_title(self._title)
        self._body = data["bodySummary"]
        self._user_name = data["ownerDisplayName"]
        self._user_url = data["url"]
        self._post_id = str(data["id"])
        self._post_site = data["siteBaseHostAddress"]
        self._post_site = self._post_site.encode("ascii", errors="replace")
        self._owner_rep = 1
        self._post_score = 0
        self._body_is_summary = True
        self._is_answer = False
        return

    def _parse_api_post(self, response):
        # post = Post(api_response={'title': '', 'body': '',
        #                           'owner': {'display_name': '', 'reputation': 1, 'link': ''},
        #                           'site': 'stackoverflow.com', 'question_id': '1', 'IsAnswer': False, 'score': 0})
        # ^ Just a ref about what an api_response can come in as.  Note that we should assume that we have each of
        #   these.

        # {'title': 'string', 'body': 'string',
        #  'owner': {'display_name': 'string', 'reputation': 1 (int), 'link': 'string'},
        #  'site': 'string', 'question_id': 'string', 'IsAnswer': False (Boolean), 'score': 0 (int), 'link': 'string',
        #  up_vote_count: 'string', down_vote_count: 'string'}

        if "title" not in response or "body" not in response:
            return

        self._title = GlobalVars.parser.unescape(response["title"])
        self._body = GlobalVars.parser.unescape(response["body"])

        if "IsAnswer" in response and response["IsAnswer"] is True:
            self._is_answer = True
        else:
            self._is_answer = False

        if "BodyIsSummary" in response and response["BodyIsSummary"] is True:
            self._body_is_summary = True
        else:
            self._body_is_summary = False

        if 'site' in response:
            self._post_site = response['site']

        if 'link' in response:
            self._post_url = response["link"]

        if 'score' in response:
            self._post_score = response["score"]

        if 'up_vote_count' in response:
            self._votes['upvotes'] = response["up_vote_count"]

        if 'down_vote_count' in response:
            self._votes['downvotes'] = response["down_vote_count"]

        if 'display_name' in response['owner']:
            self._user_name = GlobalVars.parser.unescape(response["owner"]["display_name"])

        if 'link' in response['owner']:
            self._user_url = response["owner"]["link"]

        if 'reputation' in response['owner']:
            self._owner_rep = response["owner"]["reputation"]
        else:
            self._owner_rep = 0

        # noinspection PyBroadException
        try:
            self._post_id = str(response["question_id"])
        except:
            self._post_id = str(0)

        return

    @property
    def body(self):
        return str(self._body)

    @property
    def body_is_summary(self):
        return str(self._body_is_summary)

    @property
    def is_answer(self):
        return str(self._is_answer)

    @property
    def owner_rep(self):
        return int(self._owner_rep)

    @property
    def post_id(self):
        return str(self._post_id)

    @property
    def post_score(self):
        return int(self._post_score)

    @property
    def post_site(self):
        return str(self._post_site)

    @property
    def title(self):
        return str(self._title)

    @property
    def user_name(self):
        return str(self._user_name)

    @property
    def user_url(self):
        return str(self._user_url)

    @property
    def up_vote_count(self):
        return self._votes['upvotes']

    @property
    def down_vote_count(self):
        return self._votes['downvotes']
<commit_msg>Fix unicode string issues in post.py<commit_after>import json
from globalvars import GlobalVars
import parsing

class Post:
    _body = ""
    _body_is_summary = False
    _is_answer = False
    _owner_rep = 1
    _post_id = ""
    _post_score = 0
    _post_site = ""
    _title = ""
    _user_name = ""
    _user_url = ""
    _votes = {'downvotes': None, 'upvotes': None}

    def __init__(self, json_data=None, api_response=None):
        if json_data is not None:
            self._parse_json_post(json_data)
        elif api_response is not None:
            self._parse_api_post(api_response)
        else:
            raise ValueError("Must provide either JSON Data or an API Response object for Post object.")
        # self._title = title
        # self._body = body
        # self._user_name = user_name
        # self._user_url = user_url
        # self._post_site = post_site
        # self._post_id = post_id
        # self._is_answer = is_answer
        # self._body_is_summary = body_is_summary
        # self._owner_rep = owner_rep
        # self._post_score = post_score

    def __repr__(self):
        type_name = type(self).__name__
        dataset = ['title=' + self.title, 'body=' + self.body, 'user_name=' + self.user_name,
                   'user_url=' + self.user_url, 'post_site=' + self.post_site, 'post_id=' + self.post_id,
                   'is_answer=' + str(self.is_answer), 'body_is_summary=' + str(self.body_is_summary),
                   'owner_rep=' + str(self.owner_rep), 'post_score=' + str(self.post_score)]
        return "%s(%s)" % (type_name, ', '.join(dataset))

    def _parse_json_post(self, json_data):
        text_data = json.loads(json_data)["data"]
        if text_data == "hb":
            return

        try:
            data = json.loads(text_data)
        except ValueError:
            GlobalVars.charcoal_hq.send_message(u"Encountered ValueError parsing the following:\n{0}".format(json_data),
                                                False)
            return
        if "ownerUrl" not in data:
            # owner's account doesn't exist anymore, no need to post it in chat:
            # http://chat.stackexchange.com/transcript/message/18380776#18380776
            return
        self._title = data["titleEncodedFancy"]
        self._title = parsing.unescape_title(self._title)
        self._body = data["bodySummary"]
        self._user_name = data["ownerDisplayName"]
        self._user_url = data["url"]
        self._post_id = str(data["id"])
        self._post_site = data["siteBaseHostAddress"]
        self._post_site = self._post_site.encode("ascii", errors="replace")
        self._owner_rep = 1
        self._post_score = 0
        self._body_is_summary = True
        self._is_answer = False
        return

    def _parse_api_post(self, response):
        # post = Post(api_response={'title': '', 'body': '',
        #                           'owner': {'display_name': '', 'reputation': 1, 'link': ''},
        #                           'site': 'stackoverflow.com', 'question_id': '1', 'IsAnswer': False, 'score': 0})
        # ^ Just a ref about what an api_response can come in as.  Note that we should assume that we have each of
        #   these.

        # {'title': 'string', 'body': 'string',
        #  'owner': {'display_name': 'string', 'reputation': 1 (int), 'link': 'string'},
        #  'site': 'string', 'question_id': 'string', 'IsAnswer': False (Boolean), 'score': 0 (int), 'link': 'string',
        #  up_vote_count: 'string', down_vote_count: 'string'}

        if "title" not in response or "body" not in response:
            return

        self._title = GlobalVars.parser.unescape(response["title"])
        self._body = GlobalVars.parser.unescape(response["body"])

        if "IsAnswer" in response and response["IsAnswer"] is True:
            self._is_answer = True
        else:
            self._is_answer = False

        if "BodyIsSummary" in response and response["BodyIsSummary"] is True:
            self._body_is_summary = True
        else:
            self._body_is_summary = False

        if 'site' in response:
            self._post_site = response['site']

        if 'link' in response:
            self._post_url = response["link"]

        if 'score' in response:
            self._post_score = response["score"]

        if 'up_vote_count' in response:
            self._votes['upvotes'] = response["up_vote_count"]

        if 'down_vote_count' in response:
            self._votes['downvotes'] = response["down_vote_count"]

        if 'display_name' in response['owner']:
            self._user_name = GlobalVars.parser.unescape(response["owner"]["display_name"])

        if 'link' in response['owner']:
            self._user_url = response["owner"]["link"]

        if 'reputation' in response['owner']:
            self._owner_rep = response["owner"]["reputation"]
        else:
            self._owner_rep = 0

        # noinspection PyBroadException
        try:
            self._post_id = unicode(response["question_id"])
        except:
            self._post_id = unicode(0)

        return

    @property
    def body(self):
        return unicode(self._body)

    @property
    def body_is_summary(self):
        return unicode(self._body_is_summary)

    @property
    def is_answer(self):
        return unicode(self._is_answer)

    @property
    def owner_rep(self):
        return int(self._owner_rep)

    @property
    def post_id(self):
        return unicode(self._post_id)

    @property
    def post_score(self):
        return int(self._post_score)

    @property
    def post_site(self):
        return unicode(self._post_site)

    @property
    def title(self):
        return unicode(self._title)

    @property
    def user_name(self):
        return unicode(self._user_name)

    @property
    def user_url(self):
        return unicode(self._user_url)

    @property
    def up_vote_count(self):
        return self._votes['upvotes']

    @property
    def down_vote_count(self):
        return self._votes['downvotes']
<|endoftext|>