<commit_before>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

import re

import spyne.const.xml_ns

from decimal import Decimal

from spyne.util import Break
from spyne.const.xml_ns import DEFAULT_NS

"""This module contains the ModelBase class and other building blocks for
defining models.
"""

def nillable_string(func):
    """Decorator that retuns None if input is None."""

    def wrapper(cls, string, *args, **kwargs):
        if string is None:
            return None
        else:
            return func(cls, string, *args, **kwargs)
    return wrapper

def nillable_iterable(func):
    """Decorator that retuns [] if input is None."""

    def wrapper(prot, cls, string):
        if string is None:
            return []
        else:
            return func(prot, cls, string)
    return wrapper

class AttributesMeta(type(object)):
    """I hate quirks. This is a 10-minute attempt to get rid of a one-letter
    quirk."""

    def __new__(cls, cls_name, cls_bases, cls_dict):
        # Mapper args should not be inherited.
        if not 'sqla_mapper_args' in cls_dict:
            cls_dict['sqla_mapper_args'] = None

        return type(object).__new__(cls, cls_name, cls_bases, cls_dict)

    def __init__(self, cls_name, cls_bases, cls_dict):
        nullable = cls_dict.get('nullable', None)
        nillable = cls_dict.get('nillable', None)

        assert nullable is None or nillable is None or nullable == nillable

        self.__nullable = nullable or nillable or True

        type(object).__init__(self, cls_name, cls_bases, cls_dict)

    def get_nullable(self):
        return self.__nullable
    def set_nullable(self, what):
        self.__nullable = what
    nullable = property(get_nullable, set_nullable)

    def get_nillable(self):
        return self.__nullable
    def set_nillable(self, what):
        self.__nullable = what
    nillable = property(get_nillable, set_nillable)

class ModelBase(object):
    """The base class for type markers. It defines the model interface for the
    interface generators to use and also manages class customizations that are
    mainly used for defining constraints on input values.
    """

    __orig__ = None
    """This holds the original class the class .customize()d from. """

    __extends__ = None
    """This holds the original class the class inherited or .customize()d from.
    This is different from __orig__ because it's only set when
    ``cls.is_default(cls) == False``"""

    __namespace__ = None
    """The public namespace of this class. Use ``get_namespace()`` instead of
    accessing it directly."""

    __type_name__ = None
    """The public type name of the class. Use ``get_type_name()`` instead of
    accessing it directly."""

    # These are not the xml schema defaults. The xml schema defaults are
    # considered in XmlSchema's add() method. the defaults here are to reflect
    # what people seem to want most.
    #
    # Please note that min_occurs and max_occurs must be validated in the
    # ComplexModelBase deserializer.
    class Attributes(object):
        """The class that holds the constraints for the given type."""

        __metaclass__ = AttributesMeta

        _wrapper = False
        # when skip_wrappers=True is passed to a protocol, these objects
        # are skipped. just for internal use.

        default = None
        """The default value if the input is None"""

        nillable = True
        """Set this to false to reject null values. Synonyms with
        ``nullable``."""

        min_occurs = 0
        """Set this to 1 to make this object mandatory. Can be set to any
        positive integer. Note that an object can still be null or empty, even
        if it's there."""

        max_occurs = 1
        """Can be set to any strictly positive integer. Values greater than 1
        will imply an iterable of objects as native python type. Can be set to
        ``decimal.Decimal("inf")`` for arbitrary number of arguments."""

        schema_tag = '{%s}element' % spyne.const.xml_ns.xsd
        """The tag used to add a primitives as child to a complex type in the
        xml schema."""

        translations = None
        """A dict that contains locale codes as keys and translations of field
        names to that language as values.
        """

        sub_ns = None
        """An Xml-specific attribute that specifies which namespace should be
        used for field names in classes.
        """

        sub_name = None
        """This specifies which string should be used as field name when this
        type is seriazed under a ComplexModel.
        """

        sqla_column_args = None
        """A dict that will be passed to SQLAlchemy's ``Column`` constructor as
        ``**kwargs``.
        """

        exc_mapper = False
        """If true, this field will be excluded from the table mapper of the
        parent class.
        """

        exc_table = False
        """If true, this field will be excluded from the table of the parent
        class.
        """

        exc_interface = False
        """If true, this field will be excluded from the interface document."""

        logged = True
        """If false, this object will be ignored in ``log_repr``, mostly used
        for logging purposes."""

        unique = None
        """If True, this object will be set as unique in the database schema
        with default indexing options. If the value is a string, it will be
        used as the indexing method to create the unique index. See sqlalchemy
        documentation on how to create multi-column unique constraints.
        """

        db_type = None
        """When not None, it overrides Spyne's own mapping from Spyne types to
        SQLAlchemy types. It's a standard SQLAlchemy type marker, e.g.
        ``sqlalchemy.Integer``.
        """

        table_name = None
        """Database table name."""

        xml_choice_group = None
        """When not None, shares the same <choice> tag with fields with the same
        xml_choice_group value.
        """

        index = None
        """Can be ``True``, a string, or a tuple of two strings.

        * If True, this object will be set as indexed in the database schema
          with default options.

        * If the value is a string, the value will denote the indexing method
          used by the database. See: http://www.postgresql.org/docs/9.2/static/indexes-types.html

        * If the vale is a tuple of two strings, the first value will denote the
          index name and the second value will denote the indexing method as
          above.
        """

        read_only= False
        """If True, the attribute won't be initialized from outside values."""

    class Annotations(object):
        """The class that holds the annotations for the given type."""

        __use_parent_doc__ = False
        """If set to True Annotations will use __doc__ from parent,
        This is a convenience option"""

        doc = ""
        """The public documentation for the given type."""

        appinfo = None
        """Any object that carries app-specific info."""

    class Empty(object):
        pass

    _force_own_namespace = None

    @staticmethod
    def is_default(cls):
        return True

    @classmethod
    def get_namespace_prefix(cls, interface):
        """Returns the namespace prefix for the given interface. The
        get_namespace_prefix of the interface class generates a prefix if none
        is defined.
        """

        ns = cls.get_namespace()

        retval = interface.get_namespace_prefix(ns)

        return retval

    @classmethod
    def get_namespace(cls):
        """Returns the namespace of the class. Defaults to the python module
        name."""

        return cls.__namespace__

    @staticmethod
    def resolve_namespace(cls, default_ns, tags=None):
        """This call finalizes the namespace assignment. The default namespace
        is not available until the application calls populate_interface method
        of the interface generator.
        """

        if cls.__namespace__ is spyne.const.xml_ns.DEFAULT_NS:
            cls.__namespace__ = default_ns

        if (cls.__namespace__ in spyne.const.xml_ns.const_prefmap and
                                                       not cls.is_default(cls)):
            cls.__namespace__ = default_ns

        if cls.__namespace__ is None:
            ret = []
            for f in cls.__module__.split('.'):
                if f.startswith('_'):
                    break
                else:
                    ret.append(f)

            cls.__namespace__ = '.'.join(ret)

        if cls.__namespace__ is None or len(cls.__namespace__) == 0:
            cls.__namespace__ = default_ns

        if cls.__namespace__ is None or len(cls.__namespace__) == 0:
            raise ValueError("You need to explicitly set %r.__namespace__" % cls)

    @classmethod
    def get_type_name(cls):
        """Returns the class name unless the __type_name__ attribute is defined.
        """

        retval = cls.__type_name__
        if retval is None:
            retval = cls.__name__

        return retval

    @classmethod
    def get_type_name_ns(cls, interface):
        """Returns the type name with a namespace prefix, separated by a column.
        """

        if cls.get_namespace() != None:
            return "%s:%s" % (cls.get_namespace_prefix(interface),
                                                            cls.get_type_name())

    @classmethod
    def get_element_name(cls):
        return cls.Attributes.sub_name or cls.get_type_name()

    @classmethod
    def get_element_name_ns(cls, interface):
        ns = cls.Attributes.sub_ns or cls.get_namespace()
        if ns is DEFAULT_NS:
            ns = interface.get_tns()
        if ns is not None:
            pref = interface.get_namespace_prefix(ns)
            return "%s:%s" % (pref, cls.get_element_name())

    @classmethod
    @nillable_string
    def to_string(cls, value):
        """Returns str(value). This should be overridden if this is not enough.
        """

        return str(value)

    @classmethod
    def customize(cls, **kwargs):
        """Duplicates cls and overwrites the values in ``cls.Attributes`` with
        ``**kwargs`` and returns the new class."""

        cls_name, cls_bases, cls_dict = cls._s_customize(cls, **kwargs)

        return type(cls_name, cls_bases, cls_dict)

    @staticmethod
    def _s_customize(cls, **kwargs):
        """This function duplicates and customizes the class it belongs to. The
        original class remains unchanged.

        Not meant to be overridden.
        """

        cls_dict = {}
        if getattr(cls, '__orig__', None) is None:
            cls_dict['__orig__'] = cls

        class Attributes(cls.Attributes):
            pass

        if cls.Attributes.translations is None:
            Attributes.translations = {}
        if cls.Attributes.sqla_column_args is None:
            Attributes.sqla_column_args = (), {}

        cls_dict['Attributes'] = Attributes

        # as nillable is a property, it gets reset everytime a new class is
        # defined. So we need to reinitialize it explicitly.
        Attributes.nillable = cls.Attributes.nillable

        class Annotations(cls.Annotations):
            pass
        cls_dict['Annotations'] = Annotations

        for k, v in kwargs.items():
            if k.startswith('_'):
                continue

            elif k in ("doc", "appinfo"):
                setattr(Annotations, k, v)

            elif k in ('primary_key','pk'):
                Attributes.sqla_column_args[-1]['primary_key'] = v

            elif k in ('foreign_key','fk'):
                from sqlalchemy.schema import ForeignKey
                t, d = Attributes.sqla_column_args
                fkt = (ForeignKey(v),)
                Attributes.sqla_column_args = (t + fkt, d)

            elif k in ('autoincrement', 'onupdate', 'server_default'):
                Attributes.sqla_column_args[-1][k] = v

            elif k == 'max_occurs' and v == 'unbounded':
                setattr(Attributes, k, Decimal('inf'))

            else:
                setattr(Attributes, k, v)

        return (cls.__name__, (cls,), cls_dict)

    @staticmethod
    def validate_string(cls, value):
        """Override this method to do your own input validation on the input
        string. This is called before converting the incoming string to the
        native python value."""

        return (cls.Attributes.nillable or value is not None)

    @staticmethod
    def validate_native(cls, value):
        """Override this method to do your own input validation on the native
        value. This is called after converting the incoming string to the
        native python value."""

        return (cls.Attributes.nullable or value is not None)

class Null(ModelBase):
    pass

class SimpleModel(ModelBase):
    """The base class for primitives."""

    __namespace__ = "http://www.w3.org/2001/XMLSchema"

    class Attributes(ModelBase.Attributes):
        """The class that holds the constraints for the given type."""

        class __metaclass__(AttributesMeta):
            def __init__(self, cls_name, cls_bases, cls_dict):
                AttributesMeta.__init__(self, cls_name, cls_bases, cls_dict)
                if getattr(self, '_pattern', None) is None:
                    self._pattern = None
            def get_pattern(self):
                return self._pattern
            def set_pattern(self, pattern):
                self._pattern = pattern
                if pattern is not None:
                    self._pattern_re = re.compile(pattern)
            pattern = property(get_pattern, set_pattern)

        values = set()
        """The set of possible values for this type."""

        _pattern_re = None

    def __new__(cls, **kwargs):
        """Overriden so that any attempt to instantiate a primitive will return
        a customized class instead of an instance.

        See spyne.model.base.ModelBase for more information.
        """

        retval = cls.customize(**kwargs)

        if not retval.is_default(retval):
            retval.__extends__ = cls
            retval.__type_name__ = kwargs.get("type_name", ModelBase.Empty)

        return retval

    @staticmethod
    def is_default(cls):
        return (cls.Attributes.values == SimpleModel.Attributes.values)

    @staticmethod
    def validate_string(cls, value):
        return (     ModelBase.validate_string(cls, value)
                and (len(cls.Attributes.values) == 0 or
                                                value in cls.Attributes.values)
            )

class PushBase(object):
    def __init__(self, callback, errback=None):
        self._cb = callback
        self._eb = errback

        self.length = 0
        self.ctx = None
        self.app = None
        self.response = None
        self.gen = None
        self._cb_finish = None
        self._eb_finish = None

    def _init(self, ctx, response, gen, _cb_finish, _eb_finish):
        self.length = 0

        self.ctx = ctx
        self.app = ctx.app

        self.response = response
        self.gen = gen

        self._cb_finish = _cb_finish
        self._eb_finish = _eb_finish

    def init(self, ctx, response, gen, _cb_finish, _eb_finish):
        self._init(ctx, response, gen, _cb_finish, _eb_finish)
        return self._cb(self)

    def __len__(self):
        return self.length

    def append(self, inst):
        self.gen.send(inst)
        self.length += 1

    def close(self):
        try:
            self.gen.throw(Break())
        except StopIteration:
            pass

        self._cb_finish()
<commit_msg>Couldn't override nillable flag globally<commit_after>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

import re

import spyne.const.xml_ns

from decimal import Decimal

from spyne.util import Break
from spyne.const.xml_ns import DEFAULT_NS

"""This module contains the ModelBase class and other building blocks for
defining models.
"""

def nillable_string(func):
    """Decorator that retuns None if input is None."""

    def wrapper(cls, string, *args, **kwargs):
        if string is None:
            return None
        else:
            return func(cls, string, *args, **kwargs)
    return wrapper

def nillable_iterable(func):
    """Decorator that retuns [] if input is None."""

    def wrapper(prot, cls, string):
        if string is None:
            return []
        else:
            return func(prot, cls, string)
    return wrapper

class AttributesMeta(type(object)):
    """I hate quirks. This is a 10-minute attempt to get rid of a one-letter
    quirk."""

    __nullable = True

    def __new__(cls, cls_name, cls_bases, cls_dict):
        # Mapper args should not be inherited.
        if not 'sqla_mapper_args' in cls_dict:
            cls_dict['sqla_mapper_args'] = None

        return type(object).__new__(cls, cls_name, cls_bases, cls_dict)

    def __init__(self, cls_name, cls_bases, cls_dict):
        nullable = cls_dict.get('nullable', None)
        nillable = cls_dict.get('nillable', None)

        assert nullable is None or nillable is None or nullable == nillable

        if nullable or nillable:
            self.__nullable = nullable or nillable

        type(object).__init__(self, cls_name, cls_bases, cls_dict)

    def get_nullable(self):
        return self.__nullable
    def set_nullable(self, what):
        self.__nullable = what
    nullable = property(get_nullable, set_nullable)

    def get_nillable(self):
        return self.__nullable
    def set_nillable(self, what):
        self.__nullable = what
    nillable = property(get_nillable, set_nillable)

class ModelBase(object):
    """The base class for type markers. It defines the model interface for the
    interface generators to use and also manages class customizations that are
    mainly used for defining constraints on input values.
    """

    __orig__ = None
    """This holds the original class the class .customize()d from. """

    __extends__ = None
    """This holds the original class the class inherited or .customize()d from.
    This is different from __orig__ because it's only set when
    ``cls.is_default(cls) == False``"""

    __namespace__ = None
    """The public namespace of this class. Use ``get_namespace()`` instead of
    accessing it directly."""

    __type_name__ = None
    """The public type name of the class. Use ``get_type_name()`` instead of
    accessing it directly."""

    # These are not the xml schema defaults. The xml schema defaults are
    # considered in XmlSchema's add() method. the defaults here are to reflect
    # what people seem to want most.
    #
    # Please note that min_occurs and max_occurs must be validated in the
    # ComplexModelBase deserializer.
    class Attributes(object):
        """The class that holds the constraints for the given type."""

        __metaclass__ = AttributesMeta

        _wrapper = False
        # when skip_wrappers=True is passed to a protocol, these objects
        # are skipped. just for internal use.

        default = None
        """The default value if the input is None"""

        nillable = True
        """Set this to false to reject null values. Synonyms with
        ``nullable``."""

        min_occurs = 0
        """Set this to 1 to make this object mandatory. Can be set to any
        positive integer. Note that an object can still be null or empty, even
        if it's there."""

        max_occurs = 1
        """Can be set to any strictly positive integer. Values greater than 1
        will imply an iterable of objects as native python type. Can be set to
        ``decimal.Decimal("inf")`` for arbitrary number of arguments."""

        schema_tag = '{%s}element' % spyne.const.xml_ns.xsd
        """The tag used to add a primitives as child to a complex type in the
        xml schema."""

        translations = None
        """A dict that contains locale codes as keys and translations of field
        names to that language as values.
        """

        sub_ns = None
        """An Xml-specific attribute that specifies which namespace should be
        used for field names in classes.
        """

        sub_name = None
        """This specifies which string should be used as field name when this
        type is seriazed under a ComplexModel.
        """

        sqla_column_args = None
        """A dict that will be passed to SQLAlchemy's ``Column`` constructor as
        ``**kwargs``.
        """

        exc_mapper = False
        """If true, this field will be excluded from the table mapper of the
        parent class.
        """

        exc_table = False
        """If true, this field will be excluded from the table of the parent
        class.
        """

        exc_interface = False
        """If true, this field will be excluded from the interface document."""

        logged = True
        """If false, this object will be ignored in ``log_repr``, mostly used
        for logging purposes."""

        unique = None
        """If True, this object will be set as unique in the database schema
        with default indexing options. If the value is a string, it will be
        used as the indexing method to create the unique index. See sqlalchemy
        documentation on how to create multi-column unique constraints.
        """

        db_type = None
        """When not None, it overrides Spyne's own mapping from Spyne types to
        SQLAlchemy types. It's a standard SQLAlchemy type marker, e.g.
        ``sqlalchemy.Integer``.
        """

        table_name = None
        """Database table name."""

        xml_choice_group = None
        """When not None, shares the same <choice> tag with fields with the same
        xml_choice_group value.
        """

        index = None
        """Can be ``True``, a string, or a tuple of two strings.

        * If True, this object will be set as indexed in the database schema
          with default options.

        * If the value is a string, the value will denote the indexing method
          used by the database. See: http://www.postgresql.org/docs/9.2/static/indexes-types.html

        * If the vale is a tuple of two strings, the first value will denote the
          index name and the second value will denote the indexing method as
          above.
        """

        read_only= False
        """If True, the attribute won't be initialized from outside values."""

    class Annotations(object):
        """The class that holds the annotations for the given type."""

        __use_parent_doc__ = False
        """If set to True Annotations will use __doc__ from parent,
        This is a convenience option"""

        doc = ""
        """The public documentation for the given type."""

        appinfo = None
        """Any object that carries app-specific info."""

    class Empty(object):
        pass

    _force_own_namespace = None

    @staticmethod
    def is_default(cls):
        return True

    @classmethod
    def get_namespace_prefix(cls, interface):
        """Returns the namespace prefix for the given interface. The
        get_namespace_prefix of the interface class generates a prefix if none
        is defined.
        """

        ns = cls.get_namespace()

        retval = interface.get_namespace_prefix(ns)

        return retval

    @classmethod
    def get_namespace(cls):
        """Returns the namespace of the class. Defaults to the python module
        name."""

        return cls.__namespace__

    @staticmethod
    def resolve_namespace(cls, default_ns, tags=None):
        """This call finalizes the namespace assignment. The default namespace
        is not available until the application calls populate_interface method
        of the interface generator.
        """

        if cls.__namespace__ is spyne.const.xml_ns.DEFAULT_NS:
            cls.__namespace__ = default_ns

        if (cls.__namespace__ in spyne.const.xml_ns.const_prefmap and
                                                       not cls.is_default(cls)):
            cls.__namespace__ = default_ns

        if cls.__namespace__ is None:
            ret = []
            for f in cls.__module__.split('.'):
                if f.startswith('_'):
                    break
                else:
                    ret.append(f)

            cls.__namespace__ = '.'.join(ret)

        if cls.__namespace__ is None or len(cls.__namespace__) == 0:
            cls.__namespace__ = default_ns

        if cls.__namespace__ is None or len(cls.__namespace__) == 0:
            raise ValueError("You need to explicitly set %r.__namespace__" % cls)

    @classmethod
    def get_type_name(cls):
        """Returns the class name unless the __type_name__ attribute is defined.
        """

        retval = cls.__type_name__
        if retval is None:
            retval = cls.__name__

        return retval

    @classmethod
    def get_type_name_ns(cls, interface):
        """Returns the type name with a namespace prefix, separated by a column.
        """

        if cls.get_namespace() != None:
            return "%s:%s" % (cls.get_namespace_prefix(interface),
                                                            cls.get_type_name())

    @classmethod
    def get_element_name(cls):
        return cls.Attributes.sub_name or cls.get_type_name()

    @classmethod
    def get_element_name_ns(cls, interface):
        ns = cls.Attributes.sub_ns or cls.get_namespace()
        if ns is DEFAULT_NS:
            ns = interface.get_tns()
        if ns is not None:
            pref = interface.get_namespace_prefix(ns)
            return "%s:%s" % (pref, cls.get_element_name())

    @classmethod
    @nillable_string
    def to_string(cls, value):
        """Returns str(value). This should be overridden if this is not enough.
        """

        return str(value)

    @classmethod
    def customize(cls, **kwargs):
        """Duplicates cls and overwrites the values in ``cls.Attributes`` with
        ``**kwargs`` and returns the new class."""

        cls_name, cls_bases, cls_dict = cls._s_customize(cls, **kwargs)

        return type(cls_name, cls_bases, cls_dict)

    @staticmethod
    def _s_customize(cls, **kwargs):
        """This function duplicates and customizes the class it belongs to. The
        original class remains unchanged.

        Not meant to be overridden.
        """

        cls_dict = {}
        if getattr(cls, '__orig__', None) is None:
            cls_dict['__orig__'] = cls

        class Attributes(cls.Attributes):
            pass

        if cls.Attributes.translations is None:
            Attributes.translations = {}
        if cls.Attributes.sqla_column_args is None:
            Attributes.sqla_column_args = (), {}

        cls_dict['Attributes'] = Attributes

        # as nillable is a property, it gets reset everytime a new class is
        # defined. So we need to reinitialize it explicitly.
        Attributes.nillable = cls.Attributes.nillable

        class Annotations(cls.Annotations):
            pass
        cls_dict['Annotations'] = Annotations

        for k, v in kwargs.items():
            if k.startswith('_'):
                continue

            elif k in ("doc", "appinfo"):
                setattr(Annotations, k, v)

            elif k in ('primary_key','pk'):
                Attributes.sqla_column_args[-1]['primary_key'] = v

            elif k in ('foreign_key','fk'):
                from sqlalchemy.schema import ForeignKey
                t, d = Attributes.sqla_column_args
                fkt = (ForeignKey(v),)
                Attributes.sqla_column_args = (t + fkt, d)

            elif k in ('autoincrement', 'onupdate', 'server_default'):
                Attributes.sqla_column_args[-1][k] = v

            elif k == 'max_occurs' and v == 'unbounded':
                setattr(Attributes, k, Decimal('inf'))

            else:
                setattr(Attributes, k, v)

        return (cls.__name__, (cls,), cls_dict)

    @staticmethod
    def validate_string(cls, value):
        """Override this method to do your own input validation on the input
        string. This is called before converting the incoming string to the
        native python value."""

        return (cls.Attributes.nillable or value is not None)

    @staticmethod
    def validate_native(cls, value):
        """Override this method to do your own input validation on the native
        value. This is called after converting the incoming string to the
        native python value."""

        return (cls.Attributes.nullable or value is not None)

class Null(ModelBase):
    pass

class SimpleModel(ModelBase):
    """The base class for primitives."""

    __namespace__ = "http://www.w3.org/2001/XMLSchema"

    class Attributes(ModelBase.Attributes):
        """The class that holds the constraints for the given type."""

        class __metaclass__(AttributesMeta):
            def __init__(self, cls_name, cls_bases, cls_dict):
                AttributesMeta.__init__(self, cls_name, cls_bases, cls_dict)
                if getattr(self, '_pattern', None) is None:
                    self._pattern = None
            def get_pattern(self):
                return self._pattern
            def set_pattern(self, pattern):
                self._pattern = pattern
                if pattern is not None:
                    self._pattern_re = re.compile(pattern)
            pattern = property(get_pattern, set_pattern)

        values = set()
        """The set of possible values for this type."""

        _pattern_re = None

    def __new__(cls, **kwargs):
        """Overriden so that any attempt to instantiate a primitive will return
        a customized class instead of an instance.

        See spyne.model.base.ModelBase for more information.
        """

        retval = cls.customize(**kwargs)

        if not retval.is_default(retval):
            retval.__extends__ = cls
            retval.__type_name__ = kwargs.get("type_name", ModelBase.Empty)

        return retval

    @staticmethod
    def is_default(cls):
        return (cls.Attributes.values == SimpleModel.Attributes.values)

    @staticmethod
    def validate_string(cls, value):
        return (     ModelBase.validate_string(cls, value)
                and (len(cls.Attributes.values) == 0 or
                                                value in cls.Attributes.values)
            )

class PushBase(object):
    def __init__(self, callback, errback=None):
        self._cb = callback
        self._eb = errback

        self.length = 0
        self.ctx = None
        self.app = None
        self.response = None
        self.gen = None
        self._cb_finish = None
        self._eb_finish = None

    def _init(self, ctx, response, gen, _cb_finish, _eb_finish):
        self.length = 0

        self.ctx = ctx
        self.app = ctx.app

        self.response = response
        self.gen = gen

        self._cb_finish = _cb_finish
        self._eb_finish = _eb_finish

    def init(self, ctx, response, gen, _cb_finish, _eb_finish):
        self._init(ctx, response, gen, _cb_finish, _eb_finish)
        return self._cb(self)

    def __len__(self):
        return self.length

    def append(self, inst):
        self.gen.send(inst)
        self.length += 1

    def close(self):
        try:
            self.gen.throw(Break())
        except StopIteration:
            pass

        self._cb_finish()
<|endoftext|>


<commit_before>"""
Maintain access to the schemas (input- and output file descriptions) used by the CEA. These are stored in the
``schemas.yml`` file. Reading it is supposed to be done through the :py:func:`cea.schemas.schemas`. The plugins
parameter allows reading in schemas from ``schemas.yml`` files defined in plugins.
"""

import os
import pickle
import pandas as pd
import geopandas
import yaml
import warnings
import functools
from typing import List

__author__ = "Daren Thomas"
__copyright__ = "Copyright 2020, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

# keep a cache of schemas.yml - this will never change so avoid re-reading it.
# since we can actually call schemas with a varying list of plugins, store the resulting schemas dict
# in a dict indexed by the
from cea.utilities.yaml_ordered_dict import OrderedDictYAMLLoader

__schemas = {}

def schemas(plugins):
    """Return the contents of the schemas.yml file
    :parameter plugins: the list of plugins to generate the schemas for. Use ``config.plugins`` for this.
    :type plugins: List[cea.plugin.CeaPlugin]
    """
    # loading schemas.yml is quite expensive - try to avoid it as much as possible by
    # maintaining a cache of the schemas - using the plugin list as a key
    global __schemas
    key = ":".join(str(p) for p in plugins)

    if key not in __schemas:
        # load schemas.yml from disk - here again we use a cache: a pickled version is stored in the user folder.
        schemas_yml = os.path.join(os.path.dirname(__file__), 'schemas.yml')
        schemas_pickle = os.path.expanduser("~/schemas.yml.pickle")

        def load_schemas_dict_from_yaml():
            schemas_dict = yaml.load(open(schemas_yml, "rb"), Loader=OrderedDictYAMLLoader)
            # ... so write out a pickle for next time
            with open(schemas_pickle, "wb") as schemas_pickle_fp:
                pickle.dump(schemas_dict, schemas_pickle_fp)
            return schemas_dict

        # compare the dates of the two files - use the pickle if it's newer
        schemas_dict = None
        if os.path.exists(schemas_pickle) and os.path.getmtime(schemas_pickle) > os.path.getmtime(schemas_yml):
            with open(schemas_pickle, "r") as schemas_pickle_fp:
                try:
                    schemas_dict = pickle.load(schemas_pickle_fp)
                except:
                    schemas_dict = None

        if not schemas_dict:
            # ok. this will take a while to read...
            schemas_dict = load_schemas_dict_from_yaml()

        __schemas[key] = schemas_dict

        # add the plugins - these don't use caches as their schemas.yml are (probably) much shorter
        for plugin in plugins:
            __schemas[key].update(plugin.schemas)
    return __schemas[key]

def get_schema_variables(schema):
    """
    This method returns a set of all variables within the schemas.yml. The set is organised by:
    (variable_name, locator_method, script, file_name:sheet_name)
    If the variable is from an input database, the script is replaced by "-"
    Also, if the variable is not from a tree data shape (such as xlsx or xls), the 'file_name:sheet_name' becomes 'file_name' only.
    The sheet_name is important to consider as a primary key for each variable can only be made through combining the 'file_name:sheet_name' and
    'variable_name'. Along with the locator_method, the set should contain all information necessary for most tasks.
    """

    schema_variables = set()
    for locator_method in schema:

        # if there is no script mapped to 'created_by', it must be an input_file
        # replace non-existent script with the name of the file without the extension
        if not schema[locator_method]['created_by']:
            script = "-"
        else:
            script = schema[locator_method]['created_by'][0]

        if not "schema" in schema[locator_method] or not schema[locator_method]["schema"]:
            print("Could not find schema for {locator_method}".format(locator_method=locator_method))
            continue

        # for repetitive variables, include only one instance
        for variable in schema[locator_method]['schema']:
            if variable.find('srf') != -1:
                variable = variable.replace(variable, 'srf0')
            if variable.find('PIPE') != -1:
                variable = variable.replace(variable, 'PIPE0')
            if variable.find('NODE') != -1:
                variable = variable.replace(variable, 'NODE0')
            if variable.find('B0') != -1:
                variable = variable.replace(variable, 'B001')

            # if the variable is one associated with an epw file: exclude for now
            if schema[locator_method]['file_type'] == 'epw':
                variable = 'EPW file variables'

            # if the variable is actually a sheet name due to tree data shape
            if schema[locator_method]['file_type'] in {'xlsx', 'xls'}:
                worksheet = variable
                for variable_in_sheet in schema[locator_method]['schema'][worksheet]:
                    file_name = "{file_path}:{worksheet}".format(file_path=schema[locator_method]['file_path'],
                                                                 worksheet=worksheet)
                    schema_variables.add((variable_in_sheet, locator_method, script, file_name))
            # otherwise create the meta set
            else:

                file_name = schema[locator_method]['file_path']
                schema_variables.add((variable, locator_method, script, file_name))
    return schema_variables

def get_schema_scripts(plugins):
    """Returns the list of scripts actually mentioned in the schemas.yml file"""
    schemas_dict = schemas(plugins)
    schema_scripts = set()
    for locator_method in schemas_dict:
        if schemas_dict[locator_method]['used_by']:
            for script in schemas_dict[locator_method]['used_by']:
                schema_scripts.add(script)
        if schemas_dict[locator_method]['created_by'] > 0:
            for script in schemas_dict[locator_method]['created_by']:
                schema_scripts.add(script)
    return schema_scripts

def create_schema_io(locator, lm, schema, original_function=None):
    """
    Returns a wrapper object that can be used to replace original_function - the interface remains largely
    the same.
    :param cea.inputlocator.InputLocator locator: The locator we're attaching methods to.
    :param str lm: the name of the locator method being wrapped
    :param dict schema: the configuration of this locator method as defined in schemas.yml
    :param original_function: the original locator method - so we can call it. Created from schema["file_path"] if None
    :return: SchemaIo instance
    """
    if not original_function:
        original_function = create_locator_method(lm, schema)

    file_type = schema["file_type"]
    file_type_to_schema_io = {
        "csv": CsvSchemaIo,
        "dbf": DbfSchemaIo,
        "shp": ShapefileSchemaIo,
    }
    if file_type not in file_type_to_schema_io:
        # just return the default - no read() and write() possible
        return SchemaIo(locator, lm, schema, original_function)
    return file_type_to_schema_io[file_type](locator, lm, schema, original_function)

def create_locator_method(lm, schema):
    """
    Returns a function that works as an InputLocator method - it reads the
    :param str lm: the name of the locator method
    :param dict schema: the configuration of this locator method as defined in schemas.yml
    :return: the locator method (note, only kwargs are used, if at all)
    """
    file_path = schema["file_path"]

    def locator_method(self, *args, **kwargs):
        return os.path.join(self.scenario, file_path.format(**kwargs).replace("/", os.path.sep))

    locator_method.__name__ = lm
    locator_method.__doc__ = file_path
    return locator_method

class SchemaIo(object):
    """A base class for reading and writing files using schemas.yml for validation
    The default just wraps the function - read() and write() will throw errors and should be implemented
    in subclasses
    """

    def __init__(self, locator, lm, schema, original_function):
        self.locator = locator
        self.lm = lm
        self.schema = schema
        self.original_function = original_function
        functools.update_wrapper(self, original_function)

    def __str__(self):
        return "<{class_name}({lm}): {doc}>".format(class_name=self.__class__.__name__, lm=self.lm,
                                                    doc=self.original_function.__doc__)

    def __repr__(self):
        return "<{class_name}({lm}): {doc}>".format(class_name=self.__class__.__name__, lm=self.lm,
                                                    doc=self.original_function.__doc__)

    def __call__(self, *args, **kwargs):
        return self.original_function(self.locator, *args, **kwargs)

    def read(self, *args, **kwargs):
        """
        Open the file indicated by the locator method and return it as a Dataframe.
        args and kwargs are passed to the original (undecorated) locator method to figure out the location of the
        file.

        :param args:
        :param kwargs:
        :rtype: pd.DataFrame
        """
        raise AttributeError("{lm}: don't know how to read file_type {file_type}".format(
            lm=self.lm, file_type=self.schema["file_type"]))

    def write(self, df, *args, **kwargs):
        raise AttributeError("{lm}: don't know how to write file_type {file_type}".format(
            lm=self.lm, file_type=self.schema["file_type"]))

    def new(self):
        raise AttributeError("{lm}: don't know how to create a new Dataframe for file_type {file_type}".format(
            lm=self.lm, file_type=self.schema["file_type"]))

    def validate(self, df):
        """Check to make sure the Dataframe conforms to the schema"""
        expected_columns = set(self.schema["schema"]["columns"].keys())
        found_columns = set(df.columns.values)

        # handle some extra cases
        if "PIPE0" in expected_columns:
            found_columns = {c for c in found_columns if not c.startswith("PIPE")}
            found_columns.add("PIPE0")

        # handle some extra cases
        if "NODE0" in expected_columns:
            found_columns = {c for c in found_columns if not c.startswith("NODE")}
            found_columns.add("NODE0")

        if not found_columns == expected_columns:
            missing_columns = expected_columns - found_columns
            extra_columns = found_columns - expected_columns

            warnings.warn("Dataframe does not conform to schemas.yml specification for {lm}"
                          "(missing: {missing_columns}, extra: {extra_columns}".format(
                lm=self.lm, missing_columns=missing_columns, extra_columns=extra_columns))

    def colors(self):
        """
        Returns the colors for the columns in the schema, defaulting to black if not specified. Result is a dict
        mapping column name to "rgb(219, 64, 82)" format as used in plotly.

        Note that schemas.yml specifies colors using names taken from
        :type: Dict[str, str]
        """
        from .plots.colors import color_to_rgb

        result = {}
        columns = self.schema["schema"]["columns"]
        for column in columns.keys():
            result[column] = color_to_rgb(columns[column].get("plot-color", "black"))
        return result

class CsvSchemaIo(SchemaIo):
    """Read and write csv files - and attempt to validate them."""

    def read(self, *args, **kwargs):
        """
        Open the file indicated by the locator method and return it as a Dataframe.
        args and kwargs are passed to the original (undecorated) locator method to figure out the location of the
        file.

        :param args:
        :param kwargs:
        :rtype: pd.DataFrame
        """
        df = pd.read_csv(self(*args, **kwargs))
        self.validate(df)
        return df

    def write(self, df, *args, **kwargs):
        """
        :type df: pd.Dataframe
        """
        self.validate(df)
        csv_args = {}
        if "float_format" in self.schema:
            csv_args["float_format"] = self.schema["float_format"]
        path_to_csv = self(*args, **kwargs)
        parent_folder = os.path.dirname(path_to_csv)
        if not os.path.exists(parent_folder):
            os.makedirs(parent_folder)
        df.to_csv(path_to_csv, index=False, **csv_args)

    def new(self):
        return pd.DataFrame(columns=(self.schema["schema"]["columns"].keys()))

class DbfSchemaIo(SchemaIo):
    """Read and write .dbf files - and attempt to validate them."""

    def read(self, *args, **kwargs):
        """
        Open the file indicated by the locator method and return it as a DataFrame.
        args and kwargs are passed to the original (undecorated) locator method to figure out the location of the
        file.

        :param args:
        :param kwargs:
        :rtype: pd.DataFrame
        """
        from cea.utilities.dbf import dbf_to_dataframe
        df = dbf_to_dataframe(self(*args, **kwargs))
        self.validate(df)
        return df

    def write(self, df, *args, **kwargs):
        """
        :type df: pd.Dataframe
        """
        self.validate(df)
        from cea.utilities.dbf import dataframe_to_dbf
        path_to_dbf = self(*args, **kwargs)

        parent_folder = os.path.dirname(path_to_dbf)
        if not os.path.exists(parent_folder):
            os.makedirs(parent_folder)

        dataframe_to_dbf(df, path_to_dbf)

class ShapefileSchemaIo(SchemaIo):
    """Read and write .shp files - and attempt to validate them."""

    def read(self, *args, **kwargs):
        """
        Open the file indicated by the locator method and return it as a DataFrame.
        args and kwargs are passed to the original (undecorated) locator method to figure out the location of the
        file.

        :param args:
        :param kwargs:
        :rtype: geopandas.GeoDataFrame
        """

        # make sure it's in the correct projection
        from cea.utilities.standardize_coordinates import shapefile_to_WSG_and_UTM
        df, _, __ = shapefile_to_WSG_and_UTM(self(*args, **kwargs))
        self.validate(df)

        return df

    def write(self, df, *args, **kwargs):
        """
        :type df: geopandas.GeoDataFrame
        """
        from cea.utilities.standardize_coordinates import (get_lat_lon_projected_shapefile,
                                                           get_projected_coordinate_system)
        self.validate(df)
        path_to_shp = self(*args, **kwargs)

        parent_folder = os.path.dirname(path_to_shp)
        if not os.path.exists(parent_folder):
            os.makedirs(parent_folder)
        lat, lon = get_lat_lon_projected_shapefile(df)

        # get coordinate system and re project to UTM
        df = df.to_crs(get_projected_coordinate_system(lat, lon))

        df.to_file(path_to_shp)<commit_msg>Check if schemes exists first before updating<commit_after>"""
Maintain access to the schemas (input- and output file descriptions) used by the CEA. These are stored in the
``schemas.yml`` file. Reading it is supposed to be done through the :py:func:`cea.schemas.schemas`. The plugins
parameter allows reading in schemas from ``schemas.yml`` files defined in plugins.
"""

import os
import pickle
import pandas as pd
import geopandas
import yaml
import warnings
import functools
from typing import List

__author__ = "Daren Thomas"
__copyright__ = "Copyright 2020, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

# keep a cache of schemas.yml - this will never change so avoid re-reading it.
# since we can actually call schemas with a varying list of plugins, store the resulting schemas dict
# in a dict indexed by the
from cea.utilities.yaml_ordered_dict import OrderedDictYAMLLoader

__schemas = {}

def schemas(plugins):
    """Return the contents of the schemas.yml file
    :parameter plugins: the list of plugins to generate the schemas for. Use ``config.plugins`` for this.
    :type plugins: List[cea.plugin.CeaPlugin]
    """
    # loading schemas.yml is quite expensive - try to avoid it as much as possible by
    # maintaining a cache of the schemas - using the plugin list as a key
    global __schemas
    key = ":".join(str(p) for p in plugins)

    if key not in __schemas:
        # load schemas.yml from disk - here again we use a cache: a pickled version is stored in the user folder.
        schemas_yml = os.path.join(os.path.dirname(__file__), 'schemas.yml')
        schemas_pickle = os.path.expanduser("~/schemas.yml.pickle")

        def load_schemas_dict_from_yaml():
            schemas_dict = yaml.load(open(schemas_yml, "rb"), Loader=OrderedDictYAMLLoader)
            # ... so write out a pickle for next time
            with open(schemas_pickle, "wb") as schemas_pickle_fp:
                pickle.dump(schemas_dict, schemas_pickle_fp)
            return schemas_dict

        # compare the dates of the two files - use the pickle if it's newer
        schemas_dict = None
        if os.path.exists(schemas_pickle) and os.path.getmtime(schemas_pickle) > os.path.getmtime(schemas_yml):
            with open(schemas_pickle, "r") as schemas_pickle_fp:
                try:
                    schemas_dict = pickle.load(schemas_pickle_fp)
                except:
                    schemas_dict = None

        if not schemas_dict:
            # ok. this will take a while to read...
            schemas_dict = load_schemas_dict_from_yaml()

        __schemas[key] = schemas_dict

        # add the plugins - these don't use caches as their schemas.yml are (probably) much shorter
        for plugin in plugins:
            if plugin.schemas is not None:
                __schemas[key].update(plugin.schemas)
    return __schemas[key]

def get_schema_variables(schema):
    """
    This method returns a set of all variables within the schemas.yml. The set is organised by:
    (variable_name, locator_method, script, file_name:sheet_name)
    If the variable is from an input database, the script is replaced by "-"
    Also, if the variable is not from a tree data shape (such as xlsx or xls), the 'file_name:sheet_name' becomes 'file_name' only.
    The sheet_name is important to consider as a primary key for each variable can only be made through combining the 'file_name:sheet_name' and
    'variable_name'. Along with the locator_method, the set should contain all information necessary for most tasks.
    """

    schema_variables = set()
    for locator_method in schema:

        # if there is no script mapped to 'created_by', it must be an input_file
        # replace non-existent script with the name of the file without the extension
        if not schema[locator_method]['created_by']:
            script = "-"
        else:
            script = schema[locator_method]['created_by'][0]

        if not "schema" in schema[locator_method] or not schema[locator_method]["schema"]:
            print("Could not find schema for {locator_method}".format(locator_method=locator_method))
            continue

        # for repetitive variables, include only one instance
        for variable in schema[locator_method]['schema']:
            if variable.find('srf') != -1:
                variable = variable.replace(variable, 'srf0')
            if variable.find('PIPE') != -1:
                variable = variable.replace(variable, 'PIPE0')
            if variable.find('NODE') != -1:
                variable = variable.replace(variable, 'NODE0')
            if variable.find('B0') != -1:
                variable = variable.replace(variable, 'B001')

            # if the variable is one associated with an epw file: exclude for now
            if schema[locator_method]['file_type'] == 'epw':
                variable = 'EPW file variables'

            # if the variable is actually a sheet name due to tree data shape
            if schema[locator_method]['file_type'] in {'xlsx', 'xls'}:
                worksheet = variable
                for variable_in_sheet in schema[locator_method]['schema'][worksheet]:
                    file_name = "{file_path}:{worksheet}".format(file_path=schema[locator_method]['file_path'],
                                                                 worksheet=worksheet)
                    schema_variables.add((variable_in_sheet, locator_method, script, file_name))
            # otherwise create the meta set
            else:

                file_name = schema[locator_method]['file_path']
                schema_variables.add((variable, locator_method, script, file_name))
    return schema_variables

def get_schema_scripts(plugins):
    """Returns the list of scripts actually mentioned in the schemas.yml file"""
    schemas_dict = schemas(plugins)
    schema_scripts = set()
    for locator_method in schemas_dict:
        if schemas_dict[locator_method]['used_by']:
            for script in schemas_dict[locator_method]['used_by']:
                schema_scripts.add(script)
        if schemas_dict[locator_method]['created_by'] > 0:
            for script in schemas_dict[locator_method]['created_by']:
                schema_scripts.add(script)
    return schema_scripts

def create_schema_io(locator, lm, schema, original_function=None):
    """
    Returns a wrapper object that can be used to replace original_function - the interface remains largely
    the same.
    :param cea.inputlocator.InputLocator locator: The locator we're attaching methods to.
    :param str lm: the name of the locator method being wrapped
    :param dict schema: the configuration of this locator method as defined in schemas.yml
    :param original_function: the original locator method - so we can call it. Created from schema["file_path"] if None
    :return: SchemaIo instance
    """
    if not original_function:
        original_function = create_locator_method(lm, schema)

    file_type = schema["file_type"]
    file_type_to_schema_io = {
        "csv": CsvSchemaIo,
        "dbf": DbfSchemaIo,
        "shp": ShapefileSchemaIo,
    }
    if file_type not in file_type_to_schema_io:
        # just return the default - no read() and write() possible
        return SchemaIo(locator, lm, schema, original_function)
    return file_type_to_schema_io[file_type](locator, lm, schema, original_function)

def create_locator_method(lm, schema):
    """
    Returns a function that works as an InputLocator method - it reads the
    :param str lm: the name of the locator method
    :param dict schema: the configuration of this locator method as defined in schemas.yml
    :return: the locator method (note, only kwargs are used, if at all)
    """
    file_path = schema["file_path"]

    def locator_method(self, *args, **kwargs):
        return os.path.join(self.scenario, file_path.format(**kwargs).replace("/", os.path.sep))

    locator_method.__name__ = lm
    locator_method.__doc__ = file_path
    return locator_method

class SchemaIo(object):
    """A base class for reading and writing files using schemas.yml for validation
    The default just wraps the function - read() and write() will throw errors and should be implemented
    in subclasses
    """

    def __init__(self, locator, lm, schema, original_function):
        self.locator = locator
        self.lm = lm
        self.schema = schema
        self.original_function = original_function
        functools.update_wrapper(self, original_function)

    def __str__(self):
        return "<{class_name}({lm}): {doc}>".format(class_name=self.__class__.__name__, lm=self.lm,
                                                    doc=self.original_function.__doc__)

    def __repr__(self):
        return "<{class_name}({lm}): {doc}>".format(class_name=self.__class__.__name__, lm=self.lm,
                                                    doc=self.original_function.__doc__)

    def __call__(self, *args, **kwargs):
        return self.original_function(self.locator, *args, **kwargs)

    def read(self, *args, **kwargs):
        """
        Open the file indicated by the locator method and return it as a Dataframe.
        args and kwargs are passed to the original (undecorated) locator method to figure out the location of the
        file.

        :param args:
        :param kwargs:
        :rtype: pd.DataFrame
        """
        raise AttributeError("{lm}: don't know how to read file_type {file_type}".format(
            lm=self.lm, file_type=self.schema["file_type"]))

    def write(self, df, *args, **kwargs):
        raise AttributeError("{lm}: don't know how to write file_type {file_type}".format(
            lm=self.lm, file_type=self.schema["file_type"]))

    def new(self):
        raise AttributeError("{lm}: don't know how to create a new Dataframe for file_type {file_type}".format(
            lm=self.lm, file_type=self.schema["file_type"]))

    def validate(self, df):
        """Check to make sure the Dataframe conforms to the schema"""
        expected_columns = set(self.schema["schema"]["columns"].keys())
        found_columns = set(df.columns.values)

        # handle some extra cases
        if "PIPE0" in expected_columns:
            found_columns = {c for c in found_columns if not c.startswith("PIPE")}
            found_columns.add("PIPE0")

        # handle some extra cases
        if "NODE0" in expected_columns:
            found_columns = {c for c in found_columns if not c.startswith("NODE")}
            found_columns.add("NODE0")

        if not found_columns == expected_columns:
            missing_columns = expected_columns - found_columns
            extra_columns = found_columns - expected_columns

            warnings.warn("Dataframe does not conform to schemas.yml specification for {lm}"
                          "(missing: {missing_columns}, extra: {extra_columns}".format(
                lm=self.lm, missing_columns=missing_columns, extra_columns=extra_columns))

    def colors(self):
        """
        Returns the colors for the columns in the schema, defaulting to black if not specified. Result is a dict
        mapping column name to "rgb(219, 64, 82)" format as used in plotly.

        Note that schemas.yml specifies colors using names taken from
        :type: Dict[str, str]
        """
        from .plots.colors import color_to_rgb

        result = {}
        columns = self.schema["schema"]["columns"]
        for column in columns.keys():
            result[column] = color_to_rgb(columns[column].get("plot-color", "black"))
        return result

class CsvSchemaIo(SchemaIo):
    """Read and write csv files - and attempt to validate them."""

    def read(self, *args, **kwargs):
        """
        Open the file indicated by the locator method and return it as a Dataframe.
        args and kwargs are passed to the original (undecorated) locator method to figure out the location of the
        file.

        :param args:
        :param kwargs:
        :rtype: pd.DataFrame
        """
        df = pd.read_csv(self(*args, **kwargs))
        self.validate(df)
        return df

    def write(self, df, *args, **kwargs):
        """
        :type df: pd.Dataframe
        """
        self.validate(df)
        csv_args = {}
        if "float_format" in self.schema:
            csv_args["float_format"] = self.schema["float_format"]
        path_to_csv = self(*args, **kwargs)
        parent_folder = os.path.dirname(path_to_csv)
        if not os.path.exists(parent_folder):
            os.makedirs(parent_folder)
        df.to_csv(path_to_csv, index=False, **csv_args)

    def new(self):
        return pd.DataFrame(columns=(self.schema["schema"]["columns"].keys()))

class DbfSchemaIo(SchemaIo):
    """Read and write .dbf files - and attempt to validate them."""

    def read(self, *args, **kwargs):
        """
        Open the file indicated by the locator method and return it as a DataFrame.
        args and kwargs are passed to the original (undecorated) locator method to figure out the location of the
        file.

        :param args:
        :param kwargs:
        :rtype: pd.DataFrame
        """
        from cea.utilities.dbf import dbf_to_dataframe
        df = dbf_to_dataframe(self(*args, **kwargs))
        self.validate(df)
        return df

    def write(self, df, *args, **kwargs):
        """
        :type df: pd.Dataframe
        """
        self.validate(df)
        from cea.utilities.dbf import dataframe_to_dbf
        path_to_dbf = self(*args, **kwargs)

        parent_folder = os.path.dirname(path_to_dbf)
        if not os.path.exists(parent_folder):
            os.makedirs(parent_folder)

        dataframe_to_dbf(df, path_to_dbf)

class ShapefileSchemaIo(SchemaIo):
    """Read and write .shp files - and attempt to validate them."""

    def read(self, *args, **kwargs):
        """
        Open the file indicated by the locator method and return it as a DataFrame.
        args and kwargs are passed to the original (undecorated) locator method to figure out the location of the
        file.

        :param args:
        :param kwargs:
        :rtype: geopandas.GeoDataFrame
        """

        # make sure it's in the correct projection
        from cea.utilities.standardize_coordinates import shapefile_to_WSG_and_UTM
        df, _, __ = shapefile_to_WSG_and_UTM(self(*args, **kwargs))
        self.validate(df)

        return df

    def write(self, df, *args, **kwargs):
        """
        :type df: geopandas.GeoDataFrame
        """
        from cea.utilities.standardize_coordinates import (get_lat_lon_projected_shapefile,
                                                           get_projected_coordinate_system)
        self.validate(df)
        path_to_shp = self(*args, **kwargs)

        parent_folder = os.path.dirname(path_to_shp)
        if not os.path.exists(parent_folder):
            os.makedirs(parent_folder)
        lat, lon = get_lat_lon_projected_shapefile(df)

        # get coordinate system and re project to UTM
        df = df.to_crs(get_projected_coordinate_system(lat, lon))

        df.to_file(path_to_shp)<|endoftext|>


<commit_before>"""
Provides the list of scripts known to the CEA - to be used by interfaces built on top of the CEA.
"""
from __future__ import print_function

import os
import cea
import cea.inputlocator

class CeaScript(object):
    def __init__(self, script_dict, category):
        self.name = script_dict['name']
        self.module = script_dict['module']
        self.description = script_dict.get('description', '')
        self.interfaces = script_dict.get('interfaces', ['cli'])
        self.label = script_dict.get('label', self.name)
        self.category = category
        self.parameters = script_dict.get('parameters', [])
        self.input_files = script_dict.get('input-files', [])

    def __repr__(self):
        return '<cea %s>' % self.name

    def print_script_configuration(self, config, verb='Running'):
        """
        Print a list of script parameters being used for this run of the tool. Historically, each tool
        was responsible for printing their own parameters, but that requires manually keeping track of these
        parameters.
        """
        print('City Energy Analyst version %s' % cea.__version__)
        script_name = self.name
        print("%(verb)s `cea %(script_name)s` with the following parameters:" % locals())
        for section, parameter in config.matching_parameters(self.parameters):
            section_name = section.name
            parameter_name = parameter.name
            parameter_value = parameter.get()
            print("- %(section_name)s:%(parameter_name)s = %(parameter_value)s" % locals())
            print("  (default: %s)" % parameter.default)

    def print_missing_input_files(self, config):
        schema_data = schemas()
        print()
        print("---------------------------")
        print("ERROR: Missing input files:")
        for method_name, path in self.missing_input_files(config):
            script_suggestions = (schema_data[method_name]['created_by']
                                  if 'created_by' in schema_data[method_name]
                                  else None)
            print('- {path}'.format(path=path))
            if script_suggestions:
                print('  (HINT: try running {scripts})'.format(path=path, scripts=', '.join(script_suggestions)))

    def missing_input_files(self, config):
        """
        Return a list of bound :py:class:`cea.inputlocator.InputLocator` method names, one for each file required as
        input for this script that is not present yet as well as the applied path searched for.
        :return: Sequence[str]
        """
        # get a locator without triggering the restricted to
        restricted_to = config.restricted_to
        config.restricted_to = None
        locator = cea.inputlocator.InputLocator(config.scenario)
        config.restricted_to = restricted_to

        for locator_spec in self.input_files:
            method_name, args = locator_spec[0], locator_spec[1:]
            method = getattr(locator, method_name)
            path = method(*self._lookup_args(config, locator, args))
            if not os.path.exists(path):
                yield [method_name, path]

    def _lookup_args(self, config, locator, args):
        """returns a list of arguments to a locator method"""
        result = []
        for arg in args:
            if arg == 'building_name':
                result.append(locator.get_zone_building_names()[0])
            else:
                # expect an fqname for the config object
                result.append(config.get(arg))
        return result

def _get_categories_dict():
    """Load the categories -> [script] mapping either from the YAML file or, in the case of arcgis / grasshopper,
    which don't support YAML, load from a pickled version generated on the call to ``cea install-toolbox``."""
    scripts_yml = os.path.join(os.path.dirname(__file__), 'scripts.yml')
    scripts_pickle = os.path.join(os.path.dirname(__file__), 'scripts.pickle')
    try:
        import yaml
        from cea.utilities.yaml_ordered_dict import OrderedDictYAMLLoader
        categories = yaml.load(open(scripts_yml), OrderedDictYAMLLoader)
    except ImportError:
        import pickle
        categories = pickle.load(open(scripts_pickle))
    return categories

def list_scripts():
    """List all scripts"""
    categories = _get_categories_dict()
    for category in categories.keys():
        for script_dict in categories[category]:
            yield CeaScript(script_dict, category)

def by_name(script_name):
    for script in list_scripts():
        if script.name == script_name:
            return script
    raise cea.ScriptNotFoundException('Invalid script name: %s' % script_name)

def for_interface(interface='cli'):
    """Return the list of CeaScript instances that are listed for the interface"""
    return [script for script in list_scripts() if interface in script.interfaces]

def schemas():
    """Return the contents of the schemas.yml file"""
    import yaml
    schemas_yml = os.path.join(os.path.dirname(__file__), 'schemas.yml')
    return yaml.load(open(schemas_yml))

def get_schema_variables(schema):
    """
    This method returns a set of all variables within the schema.yml. The set is organised by:
    (variable_name, locator_method, script, file_name:sheet_name)
    If the variable is from an input database, the script is replaced by "-"
    Also, if the variable is not from a tree data shape (such as xlsx or xls), the 'file_name:sheet_name' becomes 'file_name' only.
    The sheet_name is important to consider as a primary key for each variable can only be made through combining the 'file_name:sheet_name' and
    'variable_name'. Along with the locator_method, the set should contain all information necessary for most tasks.
    """

    schema_variables = set()
    for locator_method in schema:

        # if there is no script mapped to 'created_by', it must be an input_file
        # replace non-existant script with the name of the file without the extension
        if not schema[locator_method]['created_by']:
            script = "-"
        else:
            script = schema[locator_method]['created_by'][0]

        # for repetitive variables, include only one instance
        for variable in schema[locator_method]['schema']:
            if variable.find('srf') != -1:
                variable = variable.replace(variable, 'srf0')
            if variable.find('PIPE') != -1:
                variable = variable.replace(variable, 'PIPE0')
            if variable.find('NODE') != -1:
                variable = variable.replace(variable, 'NODE0')
            if variable.find('B0') != -1:
                variable = variable.replace(variable, 'B001')

            # if the variable is one associated with an epw file: exclude for now
            if schema[locator_method]['file_type'] == 'epw':
                variable = 'EPW file variables'

            # if the variable is actually a sheet name due to tree data shape
            if schema[locator_method]['file_type'] in {'xlsx', 'xls'}:
                worksheet = variable
                for variable_in_sheet in schema[locator_method]['schema'][worksheet]:
                    file_name = "{file_path}:{worksheet}".format(file_path=schema[locator_method]['file_path'],
                                                                 worksheet=worksheet)
                    schema_variables.add((variable_in_sheet, locator_method, script, file_name))
            # otherwise create the meta set
            else:

                file_name = schema[locator_method]['file_path']
                schema_variables.add((variable, locator_method, script, file_name))
    return schema_variables

def get_schema_scripts(schema):
    schema_scripts = set()
    for locator_method in schema:
        if len(schema[locator_method]['used_by']) > 0:
            for script in schema[locator_method]['used_by']:
                schema_scripts.add(script)
        if len(schema[locator_method]['created_by']) > 0:
            for script in schema[locator_method]['created_by']:
                schema_scripts.add(script)
    return schema_scripts<commit_msg>typo in comments fixed<commit_after>"""
Provides the list of scripts known to the CEA - to be used by interfaces built on top of the CEA.
"""
from __future__ import print_function

import os
import cea
import cea.inputlocator

class CeaScript(object):
    def __init__(self, script_dict, category):
        self.name = script_dict['name']
        self.module = script_dict['module']
        self.description = script_dict.get('description', '')
        self.interfaces = script_dict.get('interfaces', ['cli'])
        self.label = script_dict.get('label', self.name)
        self.category = category
        self.parameters = script_dict.get('parameters', [])
        self.input_files = script_dict.get('input-files', [])

    def __repr__(self):
        return '<cea %s>' % self.name

    def print_script_configuration(self, config, verb='Running'):
        """
        Print a list of script parameters being used for this run of the tool. Historically, each tool
        was responsible for printing their own parameters, but that requires manually keeping track of these
        parameters.
        """
        print('City Energy Analyst version %s' % cea.__version__)
        script_name = self.name
        print("%(verb)s `cea %(script_name)s` with the following parameters:" % locals())
        for section, parameter in config.matching_parameters(self.parameters):
            section_name = section.name
            parameter_name = parameter.name
            parameter_value = parameter.get()
            print("- %(section_name)s:%(parameter_name)s = %(parameter_value)s" % locals())
            print("  (default: %s)" % parameter.default)

    def print_missing_input_files(self, config):
        schema_data = schemas()
        print()
        print("---------------------------")
        print("ERROR: Missing input files:")
        for method_name, path in self.missing_input_files(config):
            script_suggestions = (schema_data[method_name]['created_by']
                                  if 'created_by' in schema_data[method_name]
                                  else None)
            print('- {path}'.format(path=path))
            if script_suggestions:
                print('  (HINT: try running {scripts})'.format(path=path, scripts=', '.join(script_suggestions)))

    def missing_input_files(self, config):
        """
        Return a list of bound :py:class:`cea.inputlocator.InputLocator` method names, one for each file required as
        input for this script that is not present yet as well as the applied path searched for.
        :return: Sequence[str]
        """
        # get a locator without triggering the restricted to
        restricted_to = config.restricted_to
        config.restricted_to = None
        locator = cea.inputlocator.InputLocator(config.scenario)
        config.restricted_to = restricted_to

        for locator_spec in self.input_files:
            method_name, args = locator_spec[0], locator_spec[1:]
            method = getattr(locator, method_name)
            path = method(*self._lookup_args(config, locator, args))
            if not os.path.exists(path):
                yield [method_name, path]

    def _lookup_args(self, config, locator, args):
        """returns a list of arguments to a locator method"""
        result = []
        for arg in args:
            if arg == 'building_name':
                result.append(locator.get_zone_building_names()[0])
            else:
                # expect an fqname for the config object
                result.append(config.get(arg))
        return result

def _get_categories_dict():
    """Load the categories -> [script] mapping either from the YAML file or, in the case of arcgis / grasshopper,
    which don't support YAML, load from a pickled version generated on the call to ``cea install-toolbox``."""
    scripts_yml = os.path.join(os.path.dirname(__file__), 'scripts.yml')
    scripts_pickle = os.path.join(os.path.dirname(__file__), 'scripts.pickle')
    try:
        import yaml
        from cea.utilities.yaml_ordered_dict import OrderedDictYAMLLoader
        categories = yaml.load(open(scripts_yml), OrderedDictYAMLLoader)
    except ImportError:
        import pickle
        categories = pickle.load(open(scripts_pickle))
    return categories

def list_scripts():
    """List all scripts"""
    categories = _get_categories_dict()
    for category in categories.keys():
        for script_dict in categories[category]:
            yield CeaScript(script_dict, category)

def by_name(script_name):
    for script in list_scripts():
        if script.name == script_name:
            return script
    raise cea.ScriptNotFoundException('Invalid script name: %s' % script_name)

def for_interface(interface='cli'):
    """Return the list of CeaScript instances that are listed for the interface"""
    return [script for script in list_scripts() if interface in script.interfaces]

def schemas():
    """Return the contents of the schemas.yml file"""
    import yaml
    schemas_yml = os.path.join(os.path.dirname(__file__), 'schemas.yml')
    return yaml.load(open(schemas_yml))

def get_schema_variables(schema):
    """
    This method returns a set of all variables within the schema.yml. The set is organised by:
    (variable_name, locator_method, script, file_name:sheet_name)
    If the variable is from an input database, the script is replaced by "-"
    Also, if the variable is not from a tree data shape (such as xlsx or xls), the 'file_name:sheet_name' becomes 'file_name' only.
    The sheet_name is important to consider as a primary key for each variable can only be made through combining the 'file_name:sheet_name' and
    'variable_name'. Along with the locator_method, the set should contain all information necessary for most tasks.
    """

    schema_variables = set()
    for locator_method in schema:

        # if there is no script mapped to 'created_by', it must be an input_file
        # replace non-existent script with the name of the file without the extension
        if not schema[locator_method]['created_by']:
            script = "-"
        else:
            script = schema[locator_method]['created_by'][0]

        # for repetitive variables, include only one instance
        for variable in schema[locator_method]['schema']:
            if variable.find('srf') != -1:
                variable = variable.replace(variable, 'srf0')
            if variable.find('PIPE') != -1:
                variable = variable.replace(variable, 'PIPE0')
            if variable.find('NODE') != -1:
                variable = variable.replace(variable, 'NODE0')
            if variable.find('B0') != -1:
                variable = variable.replace(variable, 'B001')

            # if the variable is one associated with an epw file: exclude for now
            if schema[locator_method]['file_type'] == 'epw':
                variable = 'EPW file variables'

            # if the variable is actually a sheet name due to tree data shape
            if schema[locator_method]['file_type'] in {'xlsx', 'xls'}:
                worksheet = variable
                for variable_in_sheet in schema[locator_method]['schema'][worksheet]:
                    file_name = "{file_path}:{worksheet}".format(file_path=schema[locator_method]['file_path'],
                                                                 worksheet=worksheet)
                    schema_variables.add((variable_in_sheet, locator_method, script, file_name))
            # otherwise create the meta set
            else:

                file_name = schema[locator_method]['file_path']
                schema_variables.add((variable, locator_method, script, file_name))
    return schema_variables

def get_schema_scripts(schema):
    schema_scripts = set()
    for locator_method in schema:
        if len(schema[locator_method]['used_by']) > 0:
            for script in schema[locator_method]['used_by']:
                schema_scripts.add(script)
        if len(schema[locator_method]['created_by']) > 0:
            for script in schema[locator_method]['created_by']:
                schema_scripts.add(script)
    return schema_scripts<|endoftext|>