<commit_before>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

import logging
logger = logging.getLogger(__name__)

from time import time

from collections import deque

from spyne.const.xml_ns import DEFAULT_NS
from spyne.util.oset import oset

class BODY_STYLE_WRAPPED: pass
class BODY_STYLE_EMPTY: pass
class BODY_STYLE_BARE: pass

class AuxMethodContext(object):
    """Generic object that holds information specific to auxiliary methods"""
    def __init__(self, p_ctx, error):
        self.p_ctx = p_ctx
        """Primary context that this method was bound to."""

        self.error = error
        """Error from primary context (if any)."""

class TransportContext(object):
    """Generic object that holds transport-specific context information"""
    def __init__(self, transport, type=None):
        self.itself = transport
        """The transport itself; i.e. a ServerBase instance."""

        self.type = type
        """The protocol the transport uses."""

        self.app = transport.app

        self.request_encoding = None
        """General purpose variable to hold the string identifier of a request
        encoding. It's nowadays usually 'utf-8', especially with http data"""

class ProtocolContext(object):
    """Generic object that holds transport-specific context information"""
    def __init__(self, transport, type=None):
        self.itself = transport
        """The transport itself; i.e. a ServerBase instance."""

        self.type = type
        """The protocol the transport uses."""

class EventContext(object):
    """Generic object that holds event-specific context information"""
    def __init__(self, event_id=None):
        self.event_id=event_id

class MethodContext(object):
    """The base class for all RPC Contexts. Holds all information about the
    current state of execution of a remote procedure call.
    """

    frozen = False

    @property
    def method_name(self):
        """The public name of the method the ``method_request_string`` was
        matched to.
        """

        if self.descriptor is None:
            return None
        else:
            return self.descriptor.name

    def __init__(self, transport):
        # metadata
        self.call_start = time()
        """The time the rpc operation was initiated in seconds-since-epoch
        format.

        Useful for benchmarking purposes."""

        self.call_end = None
        """The time the rpc operation was completed in seconds-since-epoch
        format.

        Useful for benchmarking purposes."""

        self.app = transport.app
        """The parent application."""

        self.udc = None
        """The user defined context. Use it to your liking."""

        self.transport = TransportContext(transport)
        """The transport-specific context. Transport implementors can use this
        to their liking."""

        self.protocol = ProtocolContext(transport)
        """The protocol-specific context. Protocol implementors can use this
        to their liking."""

        self.event = EventContext()
        """Event-specific context. Use this as you want, preferably only in
        events, as you'd probably want to separate the event data from the
        method data."""

        self.aux = None
        """Auxiliary-method specific context. You can use this to share data
        between auxiliary sessions. This is not set in primary contexts.
        """

        self.method_request_string = None
        """This is used to decide which native method to call. It is set by
        the protocol classes."""

        self.__descriptor = None

        # This is set based on the value of the descriptor.
        self.service_class = None
        """The service definition class the method belongs to."""

        #
        # Input
        #

        # stream
        self.in_string = None
        """Incoming bytestream as a sequence of ``str`` or ``bytes`` instances."""

        # parsed
        self.in_document = None
        """Incoming document, what you get when you parse the incoming stream."""
        self.in_header_doc = None
        """Incoming header document of the request."""
        self.in_body_doc = None
        """Incoming body document of the request."""

        # native
        self.in_error = None
        """Native python error object. If this is set, either there was a
        parsing error or the incoming document was representing an exception.
        """
        self.in_header = None
        """Deserialized incoming header -- a native object."""
        self.in_object = None
        """In the request (i.e. server) case, this contains the function
        argument sequence for the function in the service definition class.
        In the response (i.e. client) case, this contains the object returned
        by the remote procedure call.

        It's always a sequence of objects:
            * ``[None]`` when the function has no output (client)/input (server)
              types.
            * A single-element list that wraps the return value when the
              function has one return type defined,
            * A tuple of return values in case of the function having more than
              one return value.

        The order of the argument sequence is in line with
        ``self.descriptor.in_message._type_info.keys()``.
        """

        #
        # Output
        #

        # native
        self.out_object = None
        """In the response (i.e. server) case, this contains the native python
        object(s) returned by the function in the service definition class.
        In the request (i.e. client) case, this contains the function arguments
        passed to the function call wrapper.

        It's always a sequence of objects:
            * ``[None]`` when the function has no output (server)/input (client)
              types.
            * A single-element list that wraps the return value when the
              function has one return type defined,
            * A tuple of return values in case of the function having more than
              one return value.

        The order of the argument sequence is in line with
        ``self.descriptor.out_message._type_info.keys()``.
        """
        self.out_header = None
        """Native python object set by the function in the service definition
        class."""
        self.out_error = None
        """Native exception thrown by the function in the service definition
        class."""

        # parsed
        self.out_body_doc = None
        """Serialized body object."""
        self.out_header_doc = None
        """Serialized header object."""
        self.out_document = None
        """out_body_doc and out_header_doc wrapped in the outgoing envelope"""

        # stream
        self.out_string = None
        """The pull interface to the outgoing bytestream. It's a sequence of
        strings (which could also be a generator)."""

        #self.out_stream = None
        #"""The push interface to the outgoing bytestream. It's a file-like
        #object."""

        self.function = None
        """The callable of the user code."""

        self.locale = None
        """The locale the request will use when needed for things like date
        formatting, html rendering and such."""

        self.in_protocol = transport.app.in_protocol
        """The protocol that will be used to (de)serialize incoming input"""

        self.out_protocol = transport.app.out_protocol
        """The protocol that will be used to (de)serialize outgoing input"""

        self.frozen = True
        """When this is set, no new attribute can be added to this class
        instance. This is mostly for internal use.
        """

        self.app.event_manager.fire_event("method_context_created", self)

    def get_descriptor(self):
        return self.__descriptor

    def set_descriptor(self, descriptor):
        self.__descriptor = descriptor
        self.function = descriptor.function

    descriptor = property(get_descriptor, set_descriptor)
    """The :class:``MethodDescriptor`` object representing the current method.
    It is only set when the incoming request was successfully mapped to a method
    in the public interface. The contents of this property should not be changed
    by the user code.
    """

    def __setattr__(self, k, v):
        if self.frozen == False or k in self.__dict__ or k in \
                                                ('descriptor', 'out_protocol'):
            object.__setattr__(self, k, v)
        else:
            raise ValueError("use the udc member for storing arbitrary data "
                             "in the method context")

    def __repr__(self):
        retval = deque()
        for k, v in self.__dict__.items():
            if isinstance(v, dict):
                ret = deque(['{'])
                items = sorted(v.items())
                for k2, v2 in items:
                    ret.append('\t\t%r: %r,' % (k2, v2))
                ret.append('\t}')
                ret = '\n'.join(ret)
                retval.append("\n\t%s=%s" % (k, ret))
            else:
                retval.append("\n\t%s=%r" % (k, v))

        retval.append('\n)')

        return ''.join((self.__class__.__name__, '(', ', '.join(retval), ')'))

    def close(self):
        self.call_end = time()
        self.app.event_manager.fire_event("method_context_closed", self)

    def set_out_protocol(self, what):
        self._out_protocol = what

    def get_out_protocol(self):
        return self._out_protocol

    out_protocol = property(get_out_protocol, set_out_protocol)

class MethodDescriptor(object):
    '''This class represents the method signature of an exposed service. It is
    produced by the :func:`spyne.decorator.srpc` decorator.
    '''

    def __init__(self, function, in_message, out_message, doc,
                 is_callback=False, is_async=False, mtom=False, in_header=None,
                 out_header=None, faults=None,
                 port_type=None, no_ctx=False, udp=None, class_key=None,
                 aux=None, patterns=None, body_style=None, args=None,
                 operation_name=None, in_message_name=None):

        self.__real_function = function
        """The original callable for the user code."""

        self.reset_function()

        self.operation_name = operation_name
        """The base name of an operation without the request suffix, as generated by the
        ``@srpc`` decorator."""

        self.in_message = in_message
        """A :class:`spyne.model.complex.ComplexModel` subclass that defines the
        input signature of the user function and that was automatically
        generated by the ``@srpc`` decorator."""

        self.name = None
        """The public name of the function. Equals to the type_name of the
        in_message."""

        if body_style is BODY_STYLE_BARE:
            self.name = in_message.Attributes.sub_name

        if self.name is None:
            self.name = self.in_message.get_type_name()

        self.out_message = out_message
        """A :class:`spyne.model.complex.ComplexModel` subclass that defines the
        output signature of the user function and that was automatically
        generated by the ``@srpc`` decorator."""

        self.doc = doc
        """The function docstring."""

        # these are not working, so they are not documented.
        self.is_callback = is_callback
        self.is_async = is_async
        self.mtom = mtom
        #"""Flag to indicate whether to use MTOM transport with SOAP."""
        self.port_type = port_type
        #"""The portType this function belongs to."""

        self.in_header = in_header
        """An iterable of :class:`spyne.model.complex.ComplexModel`
        subclasses to denote the types of header objects that this method can
        accept."""

        self.out_header = out_header
        """An iterable of :class:`spyne.model.complex.ComplexModel`
        subclasses to denote the types of header objects that this method can
        emit along with its return value."""

        self.faults = faults
        """An iterable of :class:`spyne.model.fault.Fault` subclasses to denote
        the types of exceptions that this method can throw."""

        self.no_ctx = no_ctx
        """no_ctx: Boolean flag to denote whether the user code gets an
        implicit :class:`spyne.MethodContext` instance as first argument."""

        self.udp = udp
        """Short for "User Defined Properties", this is just an arbitrary python
        object set by the user to pass arbitrary metadata via the ``@srpc``
        decorator."""

        self.class_key = class_key
        """ The identifier of this method in its parent
        :class:`spyne.service.ServiceBase` subclass."""

        self.aux = aux
        """Value to indicate what kind of auxiliary method this is. (None means
        primary)

        Primary methods block the request as long as they're running. Their
        return values are returned to the client. Auxiliary ones execute
        asyncronously after the primary method returns, and their return values
        are ignored by the rpc layer.
        """

        self.patterns = patterns
        """This list stores patterns which will match this callable using
        various elements of the request protocol.

        Currently, the only object supported here is the
        :class:`spyne.protocol.http.HttpPattern` object.
        """

        self.body_style = body_style
        """One of (BODY_STYLE_EMPTY, BODY_STYLE_BARE, BODY_STYLE_WRAPPED)."""

        self.args = args
        """A sequence of the names of the exposed arguments, or None."""

    @property
    def key(self):
        """The function identifier in '{namespace}name' form."""

        assert not (self.in_message.get_namespace() is DEFAULT_NS)

        return '{%s}%s' % (
            self.in_message.get_namespace(), self.in_message.get_type_name())

    def reset_function(self, val=None):
        if val != None:
            self.__real_function = val
        self.function = self.__real_function

class EventManager(object):
    """Spyne supports a simple event system that can be used to have repetitive
    boilerplate code that has to run for every method call nicely tucked away
    in one or more event handlers. The popular use-cases include things like
    database transaction management, logging and measuring performance.

    Various Spyne components support firing events at various stages during the
    processing of a request, which are documented in the relevant classes.

    The classes that support events are:
        * :class:`spyne.application.Application`
        * :class:`spyne.service.ServiceBase`
        * :class:`spyne.protocol._base.ProtocolBase`
        * :class:`spyne.server.wsgi.WsgiApplication`

    The events are stored in an ordered set. This means that the events are ran
    in the order they were added and adding a handler twice does not cause it to
    run twice.
    """

    def __init__(self, parent, handlers={}):
        self.parent = parent
        self.handlers = dict(handlers)

    def add_listener(self, event_name, handler):
        """Register a handler for the given event name.

        :param event_name: The event identifier, indicated by the documentation.
                           Usually, this is a string.
        :param handler: A static python function that receives a single
                        MethodContext argument.
        """

        handlers = self.handlers.get(event_name, oset())
        handlers.add(handler)
        self.handlers[event_name] = handlers

    def fire_event(self, event_name, ctx):
        """Run all the handlers for a given event name.

        :param event_name: The event identifier, indicated by the documentation.
                           Usually, this is a string.
        :param ctx: The method context. Event-related data is conventionally
                        stored in ctx.event attribute.
        """

        handlers = self.handlers.get(event_name, oset())
        for handler in handlers:
            handler(ctx)
<commit_msg>remove unused var.<commit_after>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

import logging
logger = logging.getLogger(__name__)

from time import time

from collections import deque

from spyne.const.xml_ns import DEFAULT_NS
from spyne.util.oset import oset

class BODY_STYLE_WRAPPED: pass
class BODY_STYLE_EMPTY: pass
class BODY_STYLE_BARE: pass

class AuxMethodContext(object):
    """Generic object that holds information specific to auxiliary methods"""
    def __init__(self, p_ctx, error):
        self.p_ctx = p_ctx
        """Primary context that this method was bound to."""

        self.error = error
        """Error from primary context (if any)."""

class TransportContext(object):
    """Generic object that holds transport-specific context information"""
    def __init__(self, transport, type=None):
        self.itself = transport
        """The transport itself; i.e. a ServerBase instance."""

        self.type = type
        """The protocol the transport uses."""

        self.app = transport.app

        self.request_encoding = None
        """General purpose variable to hold the string identifier of a request
        encoding. It's nowadays usually 'utf-8', especially with http data"""

class ProtocolContext(object):
    """Generic object that holds transport-specific context information"""
    def __init__(self, transport, type=None):
        self.itself = transport
        """The transport itself; i.e. a ServerBase instance."""

        self.type = type
        """The protocol the transport uses."""

class EventContext(object):
    """Generic object that holds event-specific context information"""
    def __init__(self, event_id=None):
        self.event_id=event_id

class MethodContext(object):
    """The base class for all RPC Contexts. Holds all information about the
    current state of execution of a remote procedure call.
    """

    frozen = False

    @property
    def method_name(self):
        """The public name of the method the ``method_request_string`` was
        matched to.
        """

        if self.descriptor is None:
            return None
        else:
            return self.descriptor.name

    def __init__(self, transport):
        # metadata
        self.call_start = time()
        """The time the rpc operation was initiated in seconds-since-epoch
        format.

        Useful for benchmarking purposes."""

        self.call_end = None
        """The time the rpc operation was completed in seconds-since-epoch
        format.

        Useful for benchmarking purposes."""

        self.app = transport.app
        """The parent application."""

        self.udc = None
        """The user defined context. Use it to your liking."""

        self.transport = TransportContext(transport)
        """The transport-specific context. Transport implementors can use this
        to their liking."""

        self.protocol = ProtocolContext(transport)
        """The protocol-specific context. Protocol implementors can use this
        to their liking."""

        self.event = EventContext()
        """Event-specific context. Use this as you want, preferably only in
        events, as you'd probably want to separate the event data from the
        method data."""

        self.aux = None
        """Auxiliary-method specific context. You can use this to share data
        between auxiliary sessions. This is not set in primary contexts.
        """

        self.method_request_string = None
        """This is used to decide which native method to call. It is set by
        the protocol classes."""

        self.__descriptor = None

        # This is set based on the value of the descriptor.
        self.service_class = None
        """The service definition class the method belongs to."""

        #
        # Input
        #

        # stream
        self.in_string = None
        """Incoming bytestream as a sequence of ``str`` or ``bytes`` instances."""

        # parsed
        self.in_document = None
        """Incoming document, what you get when you parse the incoming stream."""
        self.in_header_doc = None
        """Incoming header document of the request."""
        self.in_body_doc = None
        """Incoming body document of the request."""

        # native
        self.in_error = None
        """Native python error object. If this is set, either there was a
        parsing error or the incoming document was representing an exception.
        """
        self.in_header = None
        """Deserialized incoming header -- a native object."""
        self.in_object = None
        """In the request (i.e. server) case, this contains the function
        argument sequence for the function in the service definition class.
        In the response (i.e. client) case, this contains the object returned
        by the remote procedure call.

        It's always a sequence of objects:
            * ``[None]`` when the function has no output (client)/input (server)
              types.
            * A single-element list that wraps the return value when the
              function has one return type defined,
            * A tuple of return values in case of the function having more than
              one return value.

        The order of the argument sequence is in line with
        ``self.descriptor.in_message._type_info.keys()``.
        """

        #
        # Output
        #

        # native
        self.out_object = None
        """In the response (i.e. server) case, this contains the native python
        object(s) returned by the function in the service definition class.
        In the request (i.e. client) case, this contains the function arguments
        passed to the function call wrapper.

        It's always a sequence of objects:
            * ``[None]`` when the function has no output (server)/input (client)
              types.
            * A single-element list that wraps the return value when the
              function has one return type defined,
            * A tuple of return values in case of the function having more than
              one return value.

        The order of the argument sequence is in line with
        ``self.descriptor.out_message._type_info.keys()``.
        """
        self.out_header = None
        """Native python object set by the function in the service definition
        class."""
        self.out_error = None
        """Native exception thrown by the function in the service definition
        class."""

        # parsed
        self.out_body_doc = None
        """Serialized body object."""
        self.out_header_doc = None
        """Serialized header object."""
        self.out_document = None
        """out_body_doc and out_header_doc wrapped in the outgoing envelope"""

        # stream
        self.out_string = None
        """The pull interface to the outgoing bytestream. It's a sequence of
        strings (which could also be a generator)."""

        #self.out_stream = None
        #"""The push interface to the outgoing bytestream. It's a file-like
        #object."""

        self.function = None
        """The callable of the user code."""

        self.locale = None
        """The locale the request will use when needed for things like date
        formatting, html rendering and such."""

        self.in_protocol = transport.app.in_protocol
        """The protocol that will be used to (de)serialize incoming input"""

        self.out_protocol = transport.app.out_protocol
        """The protocol that will be used to (de)serialize outgoing input"""

        self.frozen = True
        """When this is set, no new attribute can be added to this class
        instance. This is mostly for internal use.
        """

        self.app.event_manager.fire_event("method_context_created", self)

    def get_descriptor(self):
        return self.__descriptor

    def set_descriptor(self, descriptor):
        self.__descriptor = descriptor
        self.function = descriptor.function

    descriptor = property(get_descriptor, set_descriptor)
    """The :class:``MethodDescriptor`` object representing the current method.
    It is only set when the incoming request was successfully mapped to a method
    in the public interface. The contents of this property should not be changed
    by the user code.
    """

    def __setattr__(self, k, v):
        if self.frozen == False or k in self.__dict__ or k in \
                                                ('descriptor', 'out_protocol'):
            object.__setattr__(self, k, v)
        else:
            raise ValueError("use the udc member for storing arbitrary data "
                             "in the method context")

    def __repr__(self):
        retval = deque()
        for k, v in self.__dict__.items():
            if isinstance(v, dict):
                ret = deque(['{'])
                items = sorted(v.items())
                for k2, v2 in items:
                    ret.append('\t\t%r: %r,' % (k2, v2))
                ret.append('\t}')
                ret = '\n'.join(ret)
                retval.append("\n\t%s=%s" % (k, ret))
            else:
                retval.append("\n\t%s=%r" % (k, v))

        retval.append('\n)')

        return ''.join((self.__class__.__name__, '(', ', '.join(retval), ')'))

    def close(self):
        self.call_end = time()
        self.app.event_manager.fire_event("method_context_closed", self)

    def set_out_protocol(self, what):
        self._out_protocol = what

    def get_out_protocol(self):
        return self._out_protocol

    out_protocol = property(get_out_protocol, set_out_protocol)

class MethodDescriptor(object):
    '''This class represents the method signature of an exposed service. It is
    produced by the :func:`spyne.decorator.srpc` decorator.
    '''

    def __init__(self, function, in_message, out_message, doc,
                 is_callback=False, is_async=False, mtom=False, in_header=None,
                 out_header=None, faults=None,
                 port_type=None, no_ctx=False, udp=None, class_key=None,
                 aux=None, patterns=None, body_style=None, args=None,
                 operation_name=None):

        self.__real_function = function
        """The original callable for the user code."""

        self.reset_function()

        self.operation_name = operation_name
        """The base name of an operation without the request suffix, as generated by the
        ``@srpc`` decorator."""

        self.in_message = in_message
        """A :class:`spyne.model.complex.ComplexModel` subclass that defines the
        input signature of the user function and that was automatically
        generated by the ``@srpc`` decorator."""

        self.name = None
        """The public name of the function. Equals to the type_name of the
        in_message."""

        if body_style is BODY_STYLE_BARE:
            self.name = in_message.Attributes.sub_name

        if self.name is None:
            self.name = self.in_message.get_type_name()

        self.out_message = out_message
        """A :class:`spyne.model.complex.ComplexModel` subclass that defines the
        output signature of the user function and that was automatically
        generated by the ``@srpc`` decorator."""

        self.doc = doc
        """The function docstring."""

        # these are not working, so they are not documented.
        self.is_callback = is_callback
        self.is_async = is_async
        self.mtom = mtom
        #"""Flag to indicate whether to use MTOM transport with SOAP."""
        self.port_type = port_type
        #"""The portType this function belongs to."""

        self.in_header = in_header
        """An iterable of :class:`spyne.model.complex.ComplexModel`
        subclasses to denote the types of header objects that this method can
        accept."""

        self.out_header = out_header
        """An iterable of :class:`spyne.model.complex.ComplexModel`
        subclasses to denote the types of header objects that this method can
        emit along with its return value."""

        self.faults = faults
        """An iterable of :class:`spyne.model.fault.Fault` subclasses to denote
        the types of exceptions that this method can throw."""

        self.no_ctx = no_ctx
        """no_ctx: Boolean flag to denote whether the user code gets an
        implicit :class:`spyne.MethodContext` instance as first argument."""

        self.udp = udp
        """Short for "User Defined Properties", this is just an arbitrary python
        object set by the user to pass arbitrary metadata via the ``@srpc``
        decorator."""

        self.class_key = class_key
        """ The identifier of this method in its parent
        :class:`spyne.service.ServiceBase` subclass."""

        self.aux = aux
        """Value to indicate what kind of auxiliary method this is. (None means
        primary)

        Primary methods block the request as long as they're running. Their
        return values are returned to the client. Auxiliary ones execute
        asyncronously after the primary method returns, and their return values
        are ignored by the rpc layer.
        """

        self.patterns = patterns
        """This list stores patterns which will match this callable using
        various elements of the request protocol.

        Currently, the only object supported here is the
        :class:`spyne.protocol.http.HttpPattern` object.
        """

        self.body_style = body_style
        """One of (BODY_STYLE_EMPTY, BODY_STYLE_BARE, BODY_STYLE_WRAPPED)."""

        self.args = args
        """A sequence of the names of the exposed arguments, or None."""

    @property
    def key(self):
        """The function identifier in '{namespace}name' form."""

        assert not (self.in_message.get_namespace() is DEFAULT_NS)

        return '{%s}%s' % (
            self.in_message.get_namespace(), self.in_message.get_type_name())

    def reset_function(self, val=None):
        if val != None:
            self.__real_function = val
        self.function = self.__real_function

class EventManager(object):
    """Spyne supports a simple event system that can be used to have repetitive
    boilerplate code that has to run for every method call nicely tucked away
    in one or more event handlers. The popular use-cases include things like
    database transaction management, logging and measuring performance.

    Various Spyne components support firing events at various stages during the
    processing of a request, which are documented in the relevant classes.

    The classes that support events are:
        * :class:`spyne.application.Application`
        * :class:`spyne.service.ServiceBase`
        * :class:`spyne.protocol._base.ProtocolBase`
        * :class:`spyne.server.wsgi.WsgiApplication`

    The events are stored in an ordered set. This means that the events are ran
    in the order they were added and adding a handler twice does not cause it to
    run twice.
    """

    def __init__(self, parent, handlers={}):
        self.parent = parent
        self.handlers = dict(handlers)

    def add_listener(self, event_name, handler):
        """Register a handler for the given event name.

        :param event_name: The event identifier, indicated by the documentation.
                           Usually, this is a string.
        :param handler: A static python function that receives a single
                        MethodContext argument.
        """

        handlers = self.handlers.get(event_name, oset())
        handlers.add(handler)
        self.handlers[event_name] = handlers

    def fire_event(self, event_name, ctx):
        """Run all the handlers for a given event name.

        :param event_name: The event identifier, indicated by the documentation.
                           Usually, this is a string.
        :param ctx: The method context. Event-related data is conventionally
                        stored in ctx.event attribute.
        """

        handlers = self.handlers.get(event_name, oset())
        for handler in handlers:
            handler(ctx)
<|endoftext|>


<commit_before># *- coding: utf-8 -*-
# pylint: disable=locally-disabled, bad-continuation
# pylint: disable=wildcard-import, unused-import, unused-wildcard-import
# pylint: disable=superfluous-parens, redefined-builtin
""" Project automation for Invoke.
"""

import os
import sys
import time
import shlex
import shutil
import signal
import subprocess
import webbrowser

from rituals.easy import task, Collection, pushd
from rituals.util import antglob, notify
from rituals.util.which import which, WhichError
from rituals.acts.basic import help
from rituals.acts.documentation import upload

BASEDIR = os.path.dirname(__file__)

@task(help=dict(
    venv="Include an existing virtualenv (in '.' or in '.venv')",
    extra="Any extra patterns, space-separated and possibly quoted",
))
def clean(_dummy_ctx, venv=False, extra=''):
    """Perform house-keeping."""
    notify.banner("Cleaning up project files")

    # Add patterns based on given parameters
    venv_dirs = ['bin', 'include', 'lib', 'share', 'local', '.venv']
    patterns = ['_html/', 'pip-selfcheck.json']
    excludes = ['.git/', '.hg/', '.svn/']
    if venv:
        patterns.extend([i + '/' for i in venv_dirs])
    if extra:
        patterns.extend(shlex.split(extra))

    # Build fileset
    patterns = [antglob.includes(i) for i in patterns] + [antglob.excludes(i) for i in excludes]
    if not venv:
        # Do not scan venv dirs when not cleaning them
        patterns.extend([antglob.excludes(i + '/') for i in venv_dirs])
    fileset = antglob.FileSet(BASEDIR, patterns)

    # Iterate over matches and remove them
    for name in fileset:
        notify.info('rm {0}'.format(name))
        if name.endswith('/'):
            shutil.rmtree(os.path.join(BASEDIR, name))
        else:
            os.unlink(os.path.join(BASEDIR, name))

@task(help={
    'clean': "Start with a clean build area",
    'kill': "Kill a running watchdog process and exit",
    'opts': "Extra flags for Hovercraft",
})
def view(ctx, browse=False, clean=False, kill=False, opts=''):
    """Start live-reload watchdog."""
    def activity(what=None, i=None):
        "Helper"
        if i is None:
            sys.stdout.write(what + '\n')
        else:
            sys.stdout.write(' {}  Waiting for {}\r'.format(r'\|/-'[i % 4], what or 'something'))
        sys.stdout.flush()

    def pidof(cmd_fragment):
        """Find a process by its cmd string."""
        find_pid = "ps auxww | grep -i '{}' | grep -v grep".format(cmd_fragment.replace("'", r"'\''"))
        try:
            pid = subprocess.check_output(find_pid, shell=True).strip()
            return int(pid.split(None, 2)[1]) if pid else 0
        except subprocess.CalledProcessError:
            return 0

    # Kill old processes
    core_cmd = '''watchmedo shell-command'''
    for i in range(60):
        pid = pidof(core_cmd + '.*' + BASEDIR)
        if pid:
            print("Killing old watchdog process #{}...".format(pid))
            os.kill(pid, signal.SIGTERM)
            time.sleep(.3)
        else:
            if kill:
                print("No (more) watchdog processes for this directory found.")
            break
    if kill:
        return 0

    # Clean result and start watchdog
    index_file = '_html/index.html'
    cmd = core_cmd + (
           ''' --command="hovercraft{opts} index.rst {htmldir}" '''
           ''' --patterns="*.rst"'''
           ''' {basedir} &'''
           .format(opts=opts, htmldir=os.path.dirname(index_file), basedir=BASEDIR))
    if clean:
        ctx.run("invoke clean")
    elif os.path.exists(index_file):
        os.remove(index_file)
    subprocess.call(cmd, shell=True)

    # Wait for watchdog
    for i in range(60):
        activity('watchdog process', i)
        if pidof(core_cmd + '.*' + BASEDIR):
            activity('OK')
            break
        time.sleep(1)
    else:
        activity('ERR')
        return 1

    # Wait for HTML tree
    for i in range(60):
        activity('HTML index file', i)
        if os.path.exists(index_file):
            activity('OK')
            break
        time.sleep(1)
        # trigger first build
        if i == 0:
            os.utime(os.path.join(BASEDIR, 'README.rst'), None)
    else:
        activity('ERR')
        return 1

    # Open in browser
    webbrowser.open_new_tab(index_file)

@task(help={
    'browse': "Open slides in a new browser tab",
    'presentation': "Generate slides for presentation (with console and notes)",
})
def html(ctx, browse=False, presentation=False):
    """Build HTML tree."""
    index_file = '_html/index.html'
    opts = "--auto-console" if presentation else "-t simple --skip-notes"
    ctx.run("hovercraft {} index.rst {}".format(opts, os.path.dirname(index_file)))

    # Open in browser?
    if browse:
        webbrowser.open_new_tab(index_file)

@task(help={
    'open': "Open the generated PDF",
})
def pdf(ctx, open=False):
    """Build PDF slide show."""
    try:
        which("deck2pdf")
    except WhichError:
        notify.failure("You need to install 'deck2pdf' from https://github.com/melix/deck2pdf")
    ctx.run("invoke html")
    ctx.run("deck2pdf --profile=impressjs _html/index.html slides.pdf")
    if open:
        ctx.run("xdg-open slides.pdf &", pty=False)

@task(help={
    'open': "Open the generated image",
    'width': "Width of a single slide in pixels",
})
def thumbs(ctx, open=False, width=480):
    """Create slide show thumbnails."""
    ctx.run("invoke pdf")
    ctx.run("montage -density 300 slides.pdf -mode Concatenate -tile 2x999 -quality 80 -resize {} slides.jpg".format(width))
    if open:
        ctx.run("xdg-open slides.jpg &", pty=False)

namespace = Collection.from_module(sys.modules[__name__], name='', config={'rituals': dict(
    docs = dict(
        sources = '.',
        build = '_html',
        upload = dict(
            method = 'webdav',
            targets = dict(
                webdav = dict(url=None),  # must be set in the environment
            ),
        ),
    ),
)})
<commit_msg>kill watchdog on clean<commit_after># *- coding: utf-8 -*-
# pylint: disable=locally-disabled, bad-continuation
# pylint: disable=wildcard-import, unused-import, unused-wildcard-import
# pylint: disable=superfluous-parens, redefined-builtin
""" Project automation for Invoke.
"""

import os
import sys
import time
import shlex
import shutil
import signal
import subprocess
import webbrowser

from rituals.easy import task, Collection, pushd
from rituals.util import antglob, notify
from rituals.util.which import which, WhichError
from rituals.acts.basic import help
from rituals.acts.documentation import upload

BASEDIR = os.path.dirname(__file__)

@task(help=dict(
    venv="Include an existing virtualenv (in '.' or in '.venv')",
    extra="Any extra patterns, space-separated and possibly quoted",
))
def clean(ctx, venv=False, extra=''):
    """Perform house-keeping."""
    notify.banner("Cleaning up project files")

    ctx.run("invoke view --kill")

    # Add patterns based on given parameters
    venv_dirs = ['bin', 'include', 'lib', 'share', 'local', '.venv']
    patterns = ['_html/', 'pip-selfcheck.json']
    excludes = ['.git/', '.hg/', '.svn/']
    if venv:
        patterns.extend([i + '/' for i in venv_dirs])
    if extra:
        patterns.extend(shlex.split(extra))

    # Build fileset
    patterns = [antglob.includes(i) for i in patterns] + [antglob.excludes(i) for i in excludes]
    if not venv:
        # Do not scan venv dirs when not cleaning them
        patterns.extend([antglob.excludes(i + '/') for i in venv_dirs])
    fileset = antglob.FileSet(BASEDIR, patterns)

    # Iterate over matches and remove them
    for name in fileset:
        notify.info('rm {0}'.format(name))
        if name.endswith('/'):
            shutil.rmtree(os.path.join(BASEDIR, name))
        else:
            os.unlink(os.path.join(BASEDIR, name))

@task(help={
    'clean': "Start with a clean build area",
    'kill': "Kill a running watchdog process and exit",
    'opts': "Extra flags for Hovercraft",
})
def view(ctx, browse=False, clean=False, kill=False, opts=''):
    """Start live-reload watchdog."""
    def activity(what=None, i=None):
        "Helper"
        if i is None:
            sys.stdout.write(what + '\n')
        else:
            sys.stdout.write(' {}  Waiting for {}\r'.format(r'\|/-'[i % 4], what or 'something'))
        sys.stdout.flush()

    def pidof(cmd_fragment):
        """Find a process by its cmd string."""
        find_pid = "ps auxww | grep -i '{}' | grep -v grep".format(cmd_fragment.replace("'", r"'\''"))
        try:
            pid = subprocess.check_output(find_pid, shell=True).strip()
            return int(pid.split(None, 2)[1]) if pid else 0
        except subprocess.CalledProcessError:
            return 0

    # Kill old processes
    core_cmd = '''watchmedo shell-command'''
    for i in range(60):
        pid = pidof(core_cmd + '.*' + BASEDIR)
        if pid:
            print("Killing old watchdog process #{}...".format(pid))
            os.kill(pid, signal.SIGTERM)
            time.sleep(.3)
        else:
            if kill:
                print("No (more) watchdog processes for this directory found.")
            break
    if kill:
        return 0

    # Clean result and start watchdog
    index_file = '_html/index.html'
    cmd = core_cmd + (
           ''' --command="hovercraft{opts} index.rst {htmldir}" '''
           ''' --patterns="*.rst"'''
           ''' {basedir} &'''
           .format(opts=opts, htmldir=os.path.dirname(index_file), basedir=BASEDIR))
    if clean:
        ctx.run("invoke clean")
    elif os.path.exists(index_file):
        os.remove(index_file)
    subprocess.call(cmd, shell=True)

    # Wait for watchdog
    for i in range(60):
        activity('watchdog process', i)
        if pidof(core_cmd + '.*' + BASEDIR):
            activity('OK')
            break
        time.sleep(1)
    else:
        activity('ERR')
        return 1

    # Wait for HTML tree
    for i in range(60):
        activity('HTML index file', i)
        if os.path.exists(index_file):
            activity('OK')
            break
        time.sleep(1)
        # trigger first build
        if i == 0:
            os.utime(os.path.join(BASEDIR, 'README.rst'), None)
    else:
        activity('ERR')
        return 1

    # Open in browser
    webbrowser.open_new_tab(index_file)

@task(help={
    'browse': "Open slides in a new browser tab",
    'presentation': "Generate slides for presentation (with console and notes)",
})
def html(ctx, browse=False, presentation=False):
    """Build HTML tree."""
    index_file = '_html/index.html'
    opts = "--auto-console" if presentation else "-t simple --skip-notes"
    ctx.run("hovercraft {} index.rst {}".format(opts, os.path.dirname(index_file)))

    # Open in browser?
    if browse:
        webbrowser.open_new_tab(index_file)

@task(help={
    'open': "Open the generated PDF",
})
def pdf(ctx, open=False):
    """Build PDF slide show."""
    try:
        which("deck2pdf")
    except WhichError:
        notify.failure("You need to install 'deck2pdf' from https://github.com/melix/deck2pdf")
    ctx.run("invoke html")
    ctx.run("deck2pdf --profile=impressjs _html/index.html slides.pdf")
    if open:
        ctx.run("xdg-open slides.pdf &", pty=False)

@task(help={
    'open': "Open the generated image",
    'width': "Width of a single slide in pixels",
})
def thumbs(ctx, open=False, width=480):
    """Create slide show thumbnails."""
    ctx.run("invoke pdf")
    ctx.run("montage -density 300 slides.pdf -mode Concatenate -tile 2x999 -quality 80 -resize {} slides.jpg".format(width))
    if open:
        ctx.run("xdg-open slides.jpg &", pty=False)

namespace = Collection.from_module(sys.modules[__name__], name='', config={'rituals': dict(
    docs = dict(
        sources = '.',
        build = '_html',
        upload = dict(
            method = 'webdav',
            targets = dict(
                webdav = dict(url=None),  # must be set in the environment
            ),
        ),
    ),
)})
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
from datetime import datetime
from pytz import timezone
import pytz
from optparse import OptionParser
from dateutil import parser
import time

op_parser = OptionParser()
op_parser.add_option("-c", "--calculate-date", dest="date",
                  help="Calculate the timestamp from the date specified");

(options, args) = op_parser.parse_args()

if options.date:
	date = parser.parse(options.date)
	unix_time = int(time.mktime(date.timetuple()))
	print unix_time
	sys.exit(0);

else:
	if len(sys.argv) == 2:
		timestampString = sys.argv[1];
		timestampString = timestampString.split("-");

		#Etc/GMT have the reversed symbol, GMT-1 actually is GMT+1
		timezone = timezone("Etc/GMT-1");
	elif len(sys.argv) == 3:
		timestampString = sys.argv[1];
		timestampString = timestampString.split("-");
		tzString = sys.argv[2];
		timeZoneString ="Etc/GMT";
		#Etc/GMT have the reversed symbol, GMT-1 actually is GMT+1
		if tzString[3]=="-":
			timeZoneString+="+";
		else:
			timeZoneString+="-";
		timeZoneString+= tzString[4:]
		# ************************************************************
		timezone = timezone(timeZoneString);
	else:
		print("Wrong number of parameters");
		sys.exit(0);

	dateSt="";
	for d in timestampString:
		l = len(d);

		if l==10:
			date = datetime.fromtimestamp(int(d),timezone);
			dateSt += date.strftime('%Y-%m-%d %H:%M:%S %z')+"\t";

		elif l==13:
			date = datetime.fromtimestamp(int(d)/1e3,timezone);
			dateSt += date.strftime('%Y-%m-%d %H:%M:%S %z')+"\t";

		elif l==8:
			date = datetime.fromtimestamp(int(d)*60,timezone);
			dateSt += date.strftime('%Y-%m-%d %H:%M:%S %z')+"\t";

		elif l==6:
			date = datetime.fromtimestamp(int(d)*3600,timezone);
			dateSt += date.strftime('%Y-%m-%d %H:%M:%S %z')+" \t";

		else:
			print("Timestamp format not correctly detected");
			sys.exit(0);

	print dateSt;<commit_msg>Added support for JSON timestamp parsing<commit_after>#!/usr/bin/python
import sys
from datetime import datetime
from pytz import timezone
import pytz
from optparse import OptionParser
from dateutil import parser
import time
import simplejson as json

def parseTimestampString(timestampString,timezone):
	dateSt="";
	for d in timestampString:
		l = len(d);

		if l==10:
			date = datetime.fromtimestamp(int(d),timezone);
			dateSt += date.strftime('%Y-%m-%d %H:%M:%S %z')+"\t";

		elif l==13:
			date = datetime.fromtimestamp(int(d)/1e3,timezone);
			dateSt += date.strftime('%Y-%m-%d %H:%M:%S %z')+"\t";

		elif l==8:
			date = datetime.fromtimestamp(int(d)*60,timezone);
			dateSt += date.strftime('%Y-%m-%d %H:%M:%S %z')+"\t";

		elif l==6:
			date = datetime.fromtimestamp(int(d)*3600,timezone);
			dateSt += date.strftime('%Y-%m-%d %H:%M:%S %z')+" \t";

		else:
			print("Timestamp format not correctly detected "+d);
			sys.exit(0);

	return dateSt;

op_parser = OptionParser()
op_parser.add_option("-c", "--calculate-date", dest="date",
                  help="Calculate the timestamp from the date specified");
op_parser.add_option("-j", "--json", dest="json",
                  help="Return the conversed dates from the specified json timestamps");

(options, args) = op_parser.parse_args()

if options.date:
	date = parser.parse(options.date)
	unix_time = int(time.mktime(date.timetuple()))
	print unix_time
	sys.exit(0);

if options.json:
	#Forced GMT+0
	timezone = timezone("GMT+0");
	jsonObj = json.loads(options.json);
	timestampString="";
	try:
		for timestampPair in jsonObj[""]["LIVE"]:
			timestampString+=str(timestampPair[0])+"-";
	except:
		print "No LIVE values";

	if timestampString:
		timestampString = timestampString.split("-");
		print "************** LIVE TS ***********************\n"
		print parseTimestampString(timestampString[0:-1],timezone)+"\n";
		print "************** LIVE TS ***********************\n"

	timestampString="";
	try:
		for timestampPair in jsonObj[""]["VOD"]:
			timestampString+=str(timestampPair[0])+"-";
	except:
		print "No VOD values";
	if timestampString:
		timestampString = timestampString.split("-");
		print "************** VOD TS ***********************\n"
		print parseTimestampString(timestampString[0:-1],timezone)+"\n";
		print "************** VOD TS ***********************\n"

	sys.exit(0);

else:
	if len(sys.argv) == 2:
		timestampString = sys.argv[1];
		timestampString = timestampString.split("-");

		#Etc/GMT have the reversed symbol, GMT-1 actually is GMT+1
		timezone = timezone("Etc/GMT-1");
	elif len(sys.argv) == 3:
		timestampString = sys.argv[1];
		timestampString = timestampString.split("-");
		tzString = sys.argv[2];
		timeZoneString ="Etc/GMT";
		#Etc/GMT have the reversed symbol, GMT-1 actually is GMT+1
		if tzString[3]=="-":
			timeZoneString+="+";
		else:
			timeZoneString+="-";
		timeZoneString+= tzString[4:]
		# ************************************************************
		timezone = timezone(timeZoneString);
	else:
		print("Wrong number of parameters");
		sys.exit(0);

	print parseTimestampString(timestampString,timezone);

<|endoftext|>


<commit_before>#%%
import numpy as np
import pandas as pd
from pandas import DataFrame as Matrix

train_dt = pd.read_csv('titanic/train.csv')
test_dt = pd.read_csv('titanic/test.csv')

test_pid = test_dt['PassengerId']

#0 female 1 male 2 child
def define_person(info):
    age, sex = info
    return 2 if age < 16 else (0 if sex == 'female' else 1)

def process_data(data):
#fill na
    data['Age'] = data['Age'].fillna(int(data['Age'].median()))
    data['Fare'] = data['Fare'].fillna(int(data['Fare'].median()))
#create new member
    data['Person'] = data[['Age', 'Sex']].apply(define_person, axis=1)
#drop columns
    data = data.drop(['Sex'], axis=1)
    data = data.drop(['Cabin'], axis=1)
    data = data.drop(['Embarked'], axis=1)
    data = data.drop(['PassengerId'], axis=1)
    data = data.drop(['Name'], axis=1)
    data = data.drop(['Ticket'], axis=1)
    return data

train_dt = process_data(train_dt)
test_dt = process_data(test_dt)

X_train = train_dt.drop('Survived', axis=1)
Y_train = train_dt['Survived']
X_test = test_dt.copy()

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB

models = ({
    'logreg': LogisticRegression(),
    'svc': SVC(),
    'random forest': RandomForestClassifier(n_estimators=100),
    'knn': KNeighborsClassifier(n_neighbors=3),
    'GaussinNB': GaussianNB()
})

bestScore = 0

for name in models:
    model = models[name]
    model.fit(X_train, Y_train)
    Y_pred = model.predict(X_test)
    score = model.score(X_train, Y_train)
    print('%s = %f'%(name, score))

    if score > bestScore:
        bestScore = score
        bestPred = Y_pred

#%%
#use cv
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score

bestScore = 0
bestModel = None
for name in models:
    model = models[name]
    scores = cross_val_score(model, X_train, Y_train, cv=3, scoring='f1')
    mean = scores.mean()
    std = scores.std()
    print('mean = %0.2f, std = %0.2f'%(mean, std))

    if mean > bestScore:
        bestScore = mean
        bestModel = name

print('best model = %s, score = %0.2f'%(bestModel, bestScore))

#%%
#plot curve

paramsDef = ({
    'logreg': { 'name': 'C', 'data': [1, 3, 10, 30, 100, 300] },
    'svc': { 'name': 'C', 'data': [1, 3, 10, 30, 100, 300] },
    'random forest': { 'name': 'min_samples_split', 'data': [0.1, 0.3, 0.9, 2] },
    'knn': { 'name': 'leaf_size', 'data': [30, 60, 90, 120, 150, 180] },
    'GaussinNB': { 'name': 'priors', 'data': [None] }
})
from sklearn.model_selection import validation_curve
import matplotlib.pyplot as plt

def plot_validation_curve(title, params, paramName, t_scores, t_stds, v_scores, v_stds):
    fig = plt.figure()
    plt.title(title)
    x_axis = range(len(params))
    plt.fill_between(x_axis, t_scores - t_stds, t_scores + t_stds, alpha=0.1, color='r')
    plt.fill_between(x_axis, v_scores - v_stds, v_scores + v_stds, alpha=0.1, color='g')
    plt.plot(x_axis, t_scores, 'o-', color='r', label='Training Score')
    plt.plot(x_axis, v_scores, 'o-', color='g', label='Cross Validation Score')
    plt.ylabel('Score')
    plt.xlabel(paramName)
    plt.legend(loc='best')
    ax = fig.add_subplot(111)
    for xy in zip(x_axis, t_scores):
        ax.annotate(str(params[xy[0]]), xy=xy)

    for xy in zip(x_axis, v_scores):
        ax.annotate(str(params[xy[0]]), xy=xy)

for modelName in paramsDef:
    paramDef = paramsDef[modelName]
    paramName = paramDef['name']
    paramData = paramDef['data']
    tScores, vScores = validation_curve(models[modelName], X_train, Y_train, paramName, paramData)
    tScoresMean = np.mean(tScores, axis=1)
    tScoresStd = np.std(tScores, axis=1)
    vScoresMean = np.mean(vScores, axis=1)
    vScoresStd = np.std(vScores, axis=1)
    plot_validation_curve('%s Validation Curve'%modelName, paramData, paramName, tScoresMean, tScoresStd, vScoresMean, vScoresStd)

#%%
#plot learning curve
from sklearn.model_selection import learning_curve

def plot_learning_curve(title, ticks, t_scores, t_stds, v_scores, v_stds):
    fig = plt.figure()
    plt.title(title)
    x_axis = range(len(ticks))
    plt.fill_between(x_axis, t_scores - t_stds, t_scores + t_stds, alpha=0.1, color='r')
    plt.fill_between(x_axis, v_scores - v_stds, v_scores + v_stds, alpha=0.1, color='g')
    plt.plot(x_axis, t_scores, 'o-', color='r', label='Training Score')
    plt.plot(x_axis, v_scores, 'o-', color='g', label='Test Score')
    plt.ylabel('Score')
    plt.xlabel('Ticks')
    plt.legend(loc='best')
    ax = fig.add_subplot(111)
    for xy in zip(x_axis, t_scores):
        ax.annotate(str(ticks[xy[0]]), xy=xy)

    for xy in zip(x_axis, v_scores):
        ax.annotate(str(ticks[xy[0]]), xy=xy)

for name in models:
    model = models[name]
    tSizes, tScores, vScores = learning_curve(model, X_train, Y_train, train_sizes=np.linspace(0.1, 1.0, 5))
    tScoresMean = np.mean(tScores, axis=1)
    tScoresStd = np.std(tScores, axis=1)
    vScoresMean = np.mean(vScores, axis=1)
    vScoresStd = np.std(vScores, axis=1)
    plot_learning_curve('%s Learning Curve'%name, tSizes, tScoresMean, tScoresStd, vScoresMean, vScoresStd)
#%%
submission = Matrix({
    'PassengerId': test_pid,
    'Survived': bestPred
})

submission.to_csv('titanic/titanic.csv', index=False)<commit_msg>learning curve<commit_after>#%%
import numpy as np
import pandas as pd
from pandas import DataFrame as Matrix

train_dt = pd.read_csv('titanic/train.csv')
test_dt = pd.read_csv('titanic/test.csv')

test_pid = test_dt['PassengerId']

#0 female 1 male 2 child
def define_person(info):
    age, sex = info
    return 2 if age < 16 else (0 if sex == 'female' else 1)

def process_data(data):
#fill na
    data['Age'] = data['Age'].fillna(int(data['Age'].median()))
    data['Fare'] = data['Fare'].fillna(int(data['Fare'].median()))
#create new member
    data['Person'] = data[['Age', 'Sex']].apply(define_person, axis=1)
#drop columns
    data = data.drop(['Sex'], axis=1)
    data = data.drop(['Cabin'], axis=1)
    data = data.drop(['Embarked'], axis=1)
    data = data.drop(['PassengerId'], axis=1)
    data = data.drop(['Name'], axis=1)
    data = data.drop(['Ticket'], axis=1)
    return data

train_dt = process_data(train_dt)
test_dt = process_data(test_dt)

X_train = train_dt.drop('Survived', axis=1)
Y_train = train_dt['Survived']
X_test = test_dt.copy()

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB

models = ({
    'logreg': LogisticRegression(),
    'svc': SVC(),
    'random forest': RandomForestClassifier(n_estimators=100),
    'knn': KNeighborsClassifier(n_neighbors=3),
    'GaussinNB': GaussianNB()
})

bestScore = 0

for name in models:
    model = models[name]
    model.fit(X_train, Y_train)
    Y_pred = model.predict(X_test)
    score = model.score(X_train, Y_train)
    print('%s = %f'%(name, score))

    if score > bestScore:
        bestScore = score
        bestPred = Y_pred

#%%
#use cv
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score

bestScore = 0
bestModel = None
for name in models:
    model = models[name]
    scores = cross_val_score(model, X_train, Y_train, cv=3, scoring='f1')
    mean = scores.mean()
    std = scores.std()
    print('mean = %0.2f, std = %0.2f'%(mean, std))

    if mean > bestScore:
        bestScore = mean
        bestModel = name

print('best model = %s, score = %0.2f'%(bestModel, bestScore))

#%%
#plot curve

paramsDef = ({
    'logreg': { 'name': 'C', 'data': [1, 3, 10, 30, 100, 300] },
    'svc': { 'name': 'C', 'data': [1, 3, 10, 30, 100, 300] },
    'random forest': { 'name': 'min_samples_split', 'data': [0.1, 0.3, 0.9, 2] },
    'knn': { 'name': 'leaf_size', 'data': [30, 60, 90, 120, 150, 180] },
    'GaussinNB': { 'name': 'priors', 'data': [None] }
})
from sklearn.model_selection import validation_curve
import matplotlib.pyplot as plt

def plot_validation_curve(title, params, paramName, t_scores, t_stds, v_scores, v_stds):
    fig = plt.figure()
    plt.title(title)
    x_axis = range(len(params))
    plt.fill_between(x_axis, t_scores - t_stds, t_scores + t_stds, alpha=0.1, color='r')
    plt.fill_between(x_axis, v_scores - v_stds, v_scores + v_stds, alpha=0.1, color='g')
    plt.plot(x_axis, t_scores, 'o-', color='r', label='Training Score')
    plt.plot(x_axis, v_scores, 'o-', color='g', label='Cross Validation Score')
    plt.ylabel('Score')
    plt.xlabel(paramName)
    plt.legend(loc='best')
    ax = fig.add_subplot(111)
    for xy in zip(x_axis, t_scores):
        ax.annotate(str(params[xy[0]]), xy=xy)

    for xy in zip(x_axis, v_scores):
        ax.annotate(str(params[xy[0]]), xy=xy)

for modelName in paramsDef:
    paramDef = paramsDef[modelName]
    paramName = paramDef['name']
    paramData = paramDef['data']
    tScores, vScores = validation_curve(models[modelName], X_train, Y_train, paramName, paramData)
    tScoresMean = np.mean(tScores, axis=1)
    tScoresStd = np.std(tScores, axis=1)
    vScoresMean = np.mean(vScores, axis=1)
    vScoresStd = np.std(vScores, axis=1)
    plot_validation_curve('%s Validation Curve'%modelName, paramData, paramName, tScoresMean, tScoresStd, vScoresMean, vScoresStd)

#%%
#plot learning curve
from sklearn.model_selection import learning_curve

def plot_learning_curve(title, ticks, t_scores, t_stds, v_scores, v_stds):
    fig = plt.figure()
    plt.title(title)
    plt.fill_between(ticks, t_scores - t_stds, t_scores + t_stds, alpha=0.1, color='r')
    plt.fill_between(ticks, v_scores - v_stds, v_scores + v_stds, alpha=0.1, color='g')
    plt.plot(ticks, t_scores, 'o-', color='r', label='Training Score')
    plt.plot(ticks, v_scores, 'o-', color='g', label='Test Score')
    plt.ylabel('Score')
    plt.xlabel('Ticks')
    plt.legend(loc='best')
    ax = fig.add_subplot(111)
    for xy in zip(ticks, t_scores):
        ax.annotate(str(xy[0]), xy=xy)

    for xy in zip(ticks, v_scores):
        ax.annotate(str(xy[0]), xy=xy)

for name in models:
    model = models[name]
    tSizes, tScores, vScores = learning_curve(model, X_train, Y_train, train_sizes=np.linspace(0.1, 1.0, 5))
    tScoresMean = np.mean(tScores, axis=1)
    tScoresStd = np.std(tScores, axis=1)
    vScoresMean = np.mean(vScores, axis=1)
    vScoresStd = np.std(vScores, axis=1)
    plot_learning_curve('%s Learning Curve'%name, tSizes, tScoresMean, tScoresStd, vScoresMean, vScoresStd)
#%%
submission = Matrix({
    'PassengerId': test_pid,
    'Survived': bestPred
})

submission.to_csv('titanic/titanic.csv', index=False)<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import io
import sys
import errno
import operator
import subprocess
import functools
from tempfile import mkstemp
import logging
logger = logging.getLogger(__name__)

logdir = 'tldp-document-build-logs'

def firstfoundfile(locations):
    '''return the first existing file from a list of filenames (or None)'''
    for option in locations:
        if isreadablefile(option):
            return option
    return None

def arg_isloglevel(l):
    try:
        level = int(l)
        return level
    except ValueError:
        pass
    level = getattr(logging, l.upper(), None)
    if not level:
        level = logging.ERROR
    return level

def arg_isreadablefile(f):
    if isreadablefile(f):
        return f
    return None

def arg_isdirectory(d):
    if os.path.isdir(d):
        return d
    return None

def arg_isexecutable(f):
    if isexecutable(f):
        return f
    return None

def stem_and_ext(name):
    '''return (stem, ext) for any relative or absolute filename'''
    return os.path.splitext(os.path.basename(os.path.normpath(name)))

def execute(cmd, stdin=None, stdout=None, stderr=None,
            logdir=None, env=os.environ):
    '''(yet another) wrapper around subprocess.Popen()

    The processing tools for handling DocBook SGML, DocBook XML and Linuxdoc
    all use different conventions for writing outputs.  Some write into the
    working directory.  Others write to STDOUT.  Others accept the output file
    as a required option.

    To allow for automation and flexibility, this wrapper function does what
    most other synchronous subprocess.Popen() wrappers does, but it adds a
    feature to record the STDOUT and STDERR of the executable.  This is
    helpful when trying to diagnose build failures of individual documents.

    Required:

      - cmd: (list form only; the paranoid prefer shell=False)
        this must include the whole command-line
      - logdir: an existing directory in which temporary log files
        will be created

    Optional:

      - stdin: if not supplied, STDIN (FD 0) will be left as is
      - stdout: if not supplied, STDOUT (FD 1) will be connected
        to a named file in the logdir (and left for later inspection)
      - stderr: if not supplied, STDERR (FD 2) will be connected
        to a named file in the logdir (and left for later inspection)
      - env: if not supplied, just use current environment

    Returns: the numeric exit code of the process

    Side effects:

      * will probably create temporary files in logdir
      * function calls wait(); process execution will intentionally block
        until the child process terminates

    Possible exceptions:

      * if the first element of list cmd does not contain an executable,
        this function will raise an AssertionError
      * if logdir is not a directory, this function will raise ValueError or
        IOError
      * and, of course, any exceptions passed up from calling subprocess.Popen

    '''
    prefix = os.path.basename(cmd[0]) + '.' + str(os.getpid()) + '-'

    assert isexecutable(cmd[0])

    if logdir is None:
        raise ValueError("logdir must be a directory, cannot be None.")

    if not os.path.isdir(logdir):
        raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), logdir)

    # -- not remapping STDIN, because that doesn't make sense here
    mytfile = functools.partial(mkstemp, prefix=prefix, dir=logdir)
    if stdout is None:
        stdout, stdoutname = mytfile(suffix='.stdout')
    else:
        stdoutname = None

    if stderr is None:
        stderr, stderrname = mytfile(suffix='.stderr')
    else:
        stderrname = None

    logger.debug("About to execute: %r", cmd)
    proc = subprocess.Popen(cmd, shell=False, close_fds=True,
                            stdin=stdin, stdout=stdout, stderr=stderr,
                            env=env, preexec_fn=os.setsid)
    result = proc.wait()
    if result != 0:
        logger.warning("Return code (%s) for process: %r", result, cmd)
        logger.warning("Find STDOUT/STDERR in %s/%s*", logdir, prefix)
    if isinstance(stdout, int) and stdoutname:
        os.close(stdout)
    if isinstance(stderr, int) and stderrname:
        os.close(stderr)
    return result

def isexecutable(f):
    '''True if argument is executable'''
    return os.path.isfile(f) and os.access(f, os.X_OK)

def isreadablefile(f):
    '''True if argument is readable file'''
    return os.path.isfile(f) and os.access(f, os.R_OK)

def which(program):
    '''return None or the full path to an executable (respecting $PATH)
http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
    '''
    fpath, fname = os.path.split(program)
    if fpath and isexecutable(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            sut = os.path.join(path, program)
            if isexecutable(sut):
                return sut
    return None

def makefh(thing):
    '''return a file object; given an existing filename name or file object'''
    if isinstance(thing, io.IOBase):
        f = thing
    elif isinstance(thing, str) and os.path.isfile(thing):
        f = open(thing)
    else:
        raise TypeError("Cannot make file from %s of %r" %
                        (type(thing), thing,))
    return f

def statfile(name):
    '''return posix.stat_result (or None) for a single file name'''
    try:
        st = os.stat(name)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise e
        if os.path.islink(name):
            st = os.lstat(name)
        else:
            st = None
    return st

def statfiles(name, relative=None):
    '''return a dict() with keys being filenames and posix.stat_result values

    Required:

      name: the name should be an existing file, but accessing filesystems
            can be a racy proposition, so if the name is ENOENT, returns an
            empty dict()
            if name is a directory, os.walk() over the entire subtree and
            record and return all stat() results

    Optional:

      relative: if the filenames in the keys should be relative some other
                directory, then supply that path here (see examples)

    Bugs:
      Dealing with filesystems is always potentially a racy affair.  They go
      out for lunch sometimes.  They don't call.  They don't write.  But, at
      least we can try to rely on them as best we can--mostly, by just
      excluding any files (in the output dict()) which did not return a valid
      posix.stat_result.

    Examples:

    >>> statfiles('./docs/x509').keys()
    ['./docs/x509/tutorial.rst', './docs/x509/reference.rst', './docs/x509/index.rst']
    >>> statfiles('./docs/x509', relative='./').keys()
    ['docs/x509/reference.rst', 'docs/x509/tutorial.rst', 'docs/x509/index.rst']
    >>> statfiles('./docs/x509', relative='./docs/x509/').keys()
    ['index.rst', 'tutorial.rst', 'reference.rst']
    '''
    statinfo = dict()
    if not os.path.exists(name):
        return statinfo
    if not os.path.isdir(name):
        if relative:
            relpath = os.path.relpath(name, start=relative)
        else:
            relpath = name
        statinfo[relpath] = statfile(name)
        if statinfo[relpath] is None:
            del statinfo[relpath]
    else:
        for root, dirs, files in os.walk(name):
            inodes = list()
            inodes.extend(dirs)
            inodes.extend(files)
            for x in inodes:
                foundpath = os.path.join(root, x)
                if relative:
                    relpath = os.path.relpath(foundpath, start=relative)
                else:
                    relpath = foundpath
                statinfo[relpath] = statfile(foundpath)
                if statinfo[relpath] is None:
                    del statinfo[relpath]
    return statinfo

def att_statinfo(statinfo, attr='st_mtime', func=max):
    if statinfo:
        return func([getattr(v, attr) for v in statinfo.values()])
    else:
        return 0

max_size = functools.partial(att_statinfo, attr='st_size', func=max)
min_size = functools.partial(att_statinfo, attr='st_size', func=min)

max_mtime = functools.partial(att_statinfo, attr='st_mtime', func=max)
min_mtime = functools.partial(att_statinfo, attr='st_mtime', func=min)

max_ctime = functools.partial(att_statinfo, attr='st_ctime', func=max)
min_ctime = functools.partial(att_statinfo, attr='st_ctime', func=min)

max_atime = functools.partial(att_statinfo, attr='st_atime', func=max)
min_atime = functools.partial(att_statinfo, attr='st_atime', func=min)

def sieve(operand, statinfo, attr='st_mtime', func=operator.gt):
    result = set()
    for fname, stbuf in statinfo.items():
        if func(getattr(stbuf, attr), operand):
            result.add(fname)
    return result

mtime_gt = functools.partial(sieve, attr='st_mtime', func=operator.gt)
mtime_lt = functools.partial(sieve, attr='st_mtime', func=operator.lt)

size_gt = functools.partial(sieve, attr='st_size', func=operator.gt)
size_lt = functools.partial(sieve, attr='st_size', func=operator.lt)

#
# -- end of file
<commit_msg>spit out the contents of the STDERR, STDOUT and executed script if failure<commit_after>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import io
import sys
import errno
import operator
import subprocess
import functools
from tempfile import mkstemp
import logging
logger = logging.getLogger(__name__)

logdir = 'tldp-document-build-logs'

def firstfoundfile(locations):
    '''return the first existing file from a list of filenames (or None)'''
    for option in locations:
        if isreadablefile(option):
            return option
    return None

def arg_isloglevel(l):
    try:
        level = int(l)
        return level
    except ValueError:
        pass
    level = getattr(logging, l.upper(), None)
    if not level:
        level = logging.ERROR
    return level

def arg_isreadablefile(f):
    if isreadablefile(f):
        return f
    return None

def arg_isdirectory(d):
    if os.path.isdir(d):
        return d
    return None

def arg_isexecutable(f):
    if isexecutable(f):
        return f
    return None

def stem_and_ext(name):
    '''return (stem, ext) for any relative or absolute filename'''
    return os.path.splitext(os.path.basename(os.path.normpath(name)))

def execute(cmd, stdin=None, stdout=None, stderr=None,
            logdir=None, env=os.environ):
    '''(yet another) wrapper around subprocess.Popen()

    The processing tools for handling DocBook SGML, DocBook XML and Linuxdoc
    all use different conventions for writing outputs.  Some write into the
    working directory.  Others write to STDOUT.  Others accept the output file
    as a required option.

    To allow for automation and flexibility, this wrapper function does what
    most other synchronous subprocess.Popen() wrappers does, but it adds a
    feature to record the STDOUT and STDERR of the executable.  This is
    helpful when trying to diagnose build failures of individual documents.

    Required:

      - cmd: (list form only; the paranoid prefer shell=False)
        this must include the whole command-line
      - logdir: an existing directory in which temporary log files
        will be created

    Optional:

      - stdin: if not supplied, STDIN (FD 0) will be left as is
      - stdout: if not supplied, STDOUT (FD 1) will be connected
        to a named file in the logdir (and left for later inspection)
      - stderr: if not supplied, STDERR (FD 2) will be connected
        to a named file in the logdir (and left for later inspection)
      - env: if not supplied, just use current environment

    Returns: the numeric exit code of the process

    Side effects:

      * will probably create temporary files in logdir
      * function calls wait(); process execution will intentionally block
        until the child process terminates

    Possible exceptions:

      * if the first element of list cmd does not contain an executable,
        this function will raise an AssertionError
      * if logdir is not a directory, this function will raise ValueError or
        IOError
      * and, of course, any exceptions passed up from calling subprocess.Popen

    '''
    prefix = os.path.basename(cmd[0]) + '.' + str(os.getpid()) + '-'

    assert isexecutable(cmd[0])

    if logdir is None:
        raise ValueError("logdir must be a directory, cannot be None.")

    if not os.path.isdir(logdir):
        raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), logdir)

    # -- not remapping STDIN, because that doesn't make sense here
    mytfile = functools.partial(mkstemp, prefix=prefix, dir=logdir)
    if stdout is None:
        stdout, stdoutname = mytfile(suffix='.stdout')
    else:
        stdoutname = None

    if stderr is None:
        stderr, stderrname = mytfile(suffix='.stderr')
    else:
        stderrname = None

    logger.debug("About to execute: %r", cmd)
    proc = subprocess.Popen(cmd, shell=False, close_fds=True,
                            stdin=stdin, stdout=stdout, stderr=stderr,
                            env=env, preexec_fn=os.setsid)
    result = proc.wait()
    if result != 0:
        logger.error("Non-zero exit (%s) for process: %r", result, cmd)
        logger.error("Find STDOUT/STDERR in %s/%s*", logdir, prefix)
    if isinstance(stdout, int) and stdoutname:
        os.close(stdout)
        if result != 0:
            with open(stdoutname) as f:
                for line in f:
                    logger.debug("STDOUT: %s", line.rstrip())
    if isinstance(stderr, int) and stderrname:
        os.close(stderr)
        if result != 0:
            with open(stderrname) as f:
                for line in f:
                    logger.debug("STDERR: %s", line.rstrip())
    return result

def isexecutable(f):
    '''True if argument is executable'''
    return os.path.isfile(f) and os.access(f, os.X_OK)

def isreadablefile(f):
    '''True if argument is readable file'''
    return os.path.isfile(f) and os.access(f, os.R_OK)

def which(program):
    '''return None or the full path to an executable (respecting $PATH)
http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
    '''
    fpath, fname = os.path.split(program)
    if fpath and isexecutable(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            sut = os.path.join(path, program)
            if isexecutable(sut):
                return sut
    return None

def makefh(thing):
    '''return a file object; given an existing filename name or file object'''
    if isinstance(thing, io.IOBase):
        f = thing
    elif isinstance(thing, str) and os.path.isfile(thing):
        f = open(thing)
    else:
        raise TypeError("Cannot make file from %s of %r" %
                        (type(thing), thing,))
    return f

def statfile(name):
    '''return posix.stat_result (or None) for a single file name'''
    try:
        st = os.stat(name)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise e
        if os.path.islink(name):
            st = os.lstat(name)
        else:
            st = None
    return st

def statfiles(name, relative=None):
    '''return a dict() with keys being filenames and posix.stat_result values

    Required:

      name: the name should be an existing file, but accessing filesystems
            can be a racy proposition, so if the name is ENOENT, returns an
            empty dict()
            if name is a directory, os.walk() over the entire subtree and
            record and return all stat() results

    Optional:

      relative: if the filenames in the keys should be relative some other
                directory, then supply that path here (see examples)

    Bugs:
      Dealing with filesystems is always potentially a racy affair.  They go
      out for lunch sometimes.  They don't call.  They don't write.  But, at
      least we can try to rely on them as best we can--mostly, by just
      excluding any files (in the output dict()) which did not return a valid
      posix.stat_result.

    Examples:

    >>> statfiles('./docs/x509').keys()
    ['./docs/x509/tutorial.rst', './docs/x509/reference.rst', './docs/x509/index.rst']
    >>> statfiles('./docs/x509', relative='./').keys()
    ['docs/x509/reference.rst', 'docs/x509/tutorial.rst', 'docs/x509/index.rst']
    >>> statfiles('./docs/x509', relative='./docs/x509/').keys()
    ['index.rst', 'tutorial.rst', 'reference.rst']
    '''
    statinfo = dict()
    if not os.path.exists(name):
        return statinfo
    if not os.path.isdir(name):
        if relative:
            relpath = os.path.relpath(name, start=relative)
        else:
            relpath = name
        statinfo[relpath] = statfile(name)
        if statinfo[relpath] is None:
            del statinfo[relpath]
    else:
        for root, dirs, files in os.walk(name):
            inodes = list()
            inodes.extend(dirs)
            inodes.extend(files)
            for x in inodes:
                foundpath = os.path.join(root, x)
                if relative:
                    relpath = os.path.relpath(foundpath, start=relative)
                else:
                    relpath = foundpath
                statinfo[relpath] = statfile(foundpath)
                if statinfo[relpath] is None:
                    del statinfo[relpath]
    return statinfo

def att_statinfo(statinfo, attr='st_mtime', func=max):
    if statinfo:
        return func([getattr(v, attr) for v in statinfo.values()])
    else:
        return 0

max_size = functools.partial(att_statinfo, attr='st_size', func=max)
min_size = functools.partial(att_statinfo, attr='st_size', func=min)

max_mtime = functools.partial(att_statinfo, attr='st_mtime', func=max)
min_mtime = functools.partial(att_statinfo, attr='st_mtime', func=min)

max_ctime = functools.partial(att_statinfo, attr='st_ctime', func=max)
min_ctime = functools.partial(att_statinfo, attr='st_ctime', func=min)

max_atime = functools.partial(att_statinfo, attr='st_atime', func=max)
min_atime = functools.partial(att_statinfo, attr='st_atime', func=min)

def sieve(operand, statinfo, attr='st_mtime', func=operator.gt):
    result = set()
    for fname, stbuf in statinfo.items():
        if func(getattr(stbuf, attr), operand):
            result.add(fname)
    return result

mtime_gt = functools.partial(sieve, attr='st_mtime', func=operator.gt)
mtime_lt = functools.partial(sieve, attr='st_mtime', func=operator.lt)

size_gt = functools.partial(sieve, attr='st_size', func=operator.gt)
size_lt = functools.partial(sieve, attr='st_size', func=operator.lt)

#
# -- end of file
<|endoftext|>