<commit_before># telepathy-python - Base classes defining the interfaces of the Telepathy framework
#
# Copyright (C) 2005, 2006 Collabora Limited
# Copyright (C) 2005, 2006 Nokia Corporation
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

import dbus
import dbus.service

from telepathy.constants import (CONNECTION_HANDLE_TYPE_NONE,
                                 CHANNEL_TEXT_MESSAGE_TYPE_NORMAL)

from telepathy.errors import InvalidArgument

from telepathy.interfaces import (CHANNEL_INTERFACE,
                                  CHANNEL_INTERFACE_DTMF,
                                  CHANNEL_INTERFACE_GROUP,
                                  CHANNEL_INTERFACE_HOLD,
                                  CHANNEL_INTERFACE_PASSWORD,
                                  CHANNEL_TYPE_CONTACT_LIST,
                                  CHANNEL_TYPE_FILE_TRANSFER,
                                  CHANNEL_TYPE_ROOM_LIST,
                                  CHANNEL_TYPE_STREAMED_MEDIA,
                                  CHANNEL_TYPE_TEXT,
                                  MEDIA_SESSION_HANDLER,
                                  MEDIA_STREAM_HANDLER)

from telepathy._generated.Channel import Channel as _Channel

from telepathy.server.properties import DBusProperties

class Channel(_Channel, DBusProperties):

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the base channel object.

        Parameters:
        connection - the parent Connection object
        props - initial channel properties
        """
        self._conn = connection
        self._chan_manager = manager

        object_path = self._conn.get_channel_path(object_path)
        _Channel.__init__(self, self._conn._name, object_path)

        self._type = props[CHANNEL_INTERFACE + '.ChannelType']
        self._requested = props[CHANNEL_INTERFACE + '.Requested']

        self._immutable_properties = dict()

        self._handle = self._conn.handle(
            props[CHANNEL_INTERFACE + '.TargetHandleType'],
            props[CHANNEL_INTERFACE + '.TargetHandle'])
        self._interfaces = set()

        DBusProperties.__init__(self)
        self._implement_property_get(CHANNEL_INTERFACE,
            {'ChannelType': lambda: dbus.String(self.GetChannelType()),
             'Interfaces': lambda: dbus.Array(self.GetInterfaces(), signature='s'),
             'TargetHandle': lambda: dbus.UInt32(self._get_target_handle()),
             'TargetHandleType': lambda: dbus.UInt32(self._get_handle_type()),
             'TargetID': lambda: dbus.String(self._get_target_id()),
             'Requested': lambda: self._requested})

        self._add_immutables({
            'ChannelType': CHANNEL_INTERFACE,
            'TargetHandle': CHANNEL_INTERFACE,
            'Interfaces': CHANNEL_INTERFACE,
            'TargetHandleType': CHANNEL_INTERFACE,
            'TargetID': CHANNEL_INTERFACE,
            'Requested': CHANNEL_INTERFACE
            })

    def _add_immutables(self, props):
        self._immutable_properties.update(props)

    def _get_target_handle(self):
        if self._handle:
            return self._handle.get_id()
        else:
            return 0

    def _get_handle_type(self):
        if self._handle:
            return self._handle.get_type()
        else:
            return CONNECTION_HANDLE_TYPE_NONE

    def _get_target_id(self):
        if self._handle:
            return self._handle.get_name()
        else:
            return ''

    def get_props(self):
        props = dict()
        for prop, iface in self._immutable_properties.items():
            props[iface + '.' + prop] = \
                self._prop_getters[iface][prop]()
        return props

    @dbus.service.method(CHANNEL_INTERFACE, in_signature='', out_signature='')
    def Close(self):
        self.Closed()
        self._chan_manager.remove_channel(self)
        self._conn.remove_channel(self)

    @dbus.service.method(CHANNEL_INTERFACE, in_signature='', out_signature='s')
    def GetChannelType(self):
        """ Returns the interface name for the type of this channel. """
        return self._type

    @dbus.service.method(CHANNEL_INTERFACE, in_signature='', out_signature='uu')
    def GetHandle(self):
        """ Returns the handle type and number if this channel represents a
        communication with a particular contact, room or server-stored list, or
        zero if it is transient and defined only by its contents. """
        if self._handle:
            return self._handle.get_type(), self._handle
        else:
            return (CONNECTION_HANDLE_TYPE_NONE, 0)

    @dbus.service.method(CHANNEL_INTERFACE, in_signature='', out_signature='as')
    def GetInterfaces(self):
        """
        Get the optional interfaces implemented by the channel.

        Returns:
        an array of the D-Bus interface names
        """
        return self._interfaces

from telepathy._generated.Channel_Type_Contact_List \
        import ChannelTypeContactList as _ChannelTypeContactListIface

class ChannelTypeContactList(Channel, _ChannelTypeContactListIface):
    __doc__ = _ChannelTypeContactListIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)

from telepathy._generated.Channel_Type_File_Transfer \
        import ChannelTypeFileTransfer as _ChannelTypeFileTransferIface

class ChannelTypeFileTransfer(Channel, _ChannelTypeFileTransferIface):
    __doc__ = _ChannelTypeFileTransferIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)

from telepathy._generated.Channel_Type_Streamed_Media \
        import ChannelTypeStreamedMedia as _ChannelTypeStreamedMediaIface

class ChannelTypeStreamedMedia(Channel, _ChannelTypeStreamedMediaIface):
    __doc__ = _ChannelTypeStreamedMediaIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)

from telepathy._generated.Channel_Type_Room_List \
        import ChannelTypeRoomList as _ChannelTypeRoomListIface

class ChannelTypeRoomList(Channel, _ChannelTypeRoomListIface):
    __doc__ = _ChannelTypeRoomListIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)
        self._listing_rooms = False
        self._rooms = {}

        self._add_immutables(self, {'Server': CHANNEL_TYPE_ROOM_LIST})

    @dbus.service.method(CHANNEL_TYPE_ROOM_LIST, in_signature='', out_signature='b')
    def GetListingRooms(self):
        return self._listing_rooms

    @dbus.service.signal(CHANNEL_TYPE_ROOM_LIST, signature='b')
    def ListingRooms(self, listing):
        self._listing_rooms = listing

from telepathy._generated.Channel_Type_Text \
        import ChannelTypeText as _ChannelTypeTextIface

class ChannelTypeText(Channel, _ChannelTypeTextIface):
    __doc__ = _ChannelTypeTextIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)

        self._pending_messages = {}
        self._message_types = [CHANNEL_TEXT_MESSAGE_TYPE_NORMAL]

    @dbus.service.method(CHANNEL_TYPE_TEXT, in_signature='', out_signature='au')
    def GetMessageTypes(self):
        """
        Return an array indicating which types of message may be sent on this
        channel.

        Returns:
        an array of integer message types as defined above
        """
        return self._message_types

    @dbus.service.method(CHANNEL_TYPE_TEXT, in_signature='au', out_signature='')
    def AcknowledgePendingMessages(self, ids):
        """
        Inform the channel that you have handled messages by displaying them to
        the user (or equivalent), so they can be removed from the pending queue.

        Parameters:
        ids - the message to acknowledge

        Possible Errors:
        InvalidArgument (a given message ID was not found, no action taken)
        """
        for id in ids:
            if id not in self._pending_messages:
                raise InvalidArgument("the given message ID was not found")

        for id in ids:
            del self._pending_messages[id]

    @dbus.service.method(CHANNEL_TYPE_TEXT, in_signature='b', out_signature='a(uuuuus)')
    def ListPendingMessages(self, clear):
        """
        List the messages currently in the pending queue, and optionally
        remove then all.

        Parameters:
        clear - a boolean indicating whether the queue should be cleared

        Returns:
        an array of structs containing:
            a numeric identifier
            a unix timestamp indicating when the message was received
            an integer handle of the contact who sent the message
            an integer of the message type
            a bitwise OR of the message flags
            a string of the text of the message
        """
        messages = []
        for id in self._pending_messages.keys():
            (timestamp, sender, type, flags, text) = self._pending_messages[id]
            message = (id, timestamp, sender, type, flags, text)
            messages.append(message)
            if clear:
                del self._pending_messages[id]
        messages.sort(cmp=lambda x,y:cmp(x[1], y[1]))
        return messages

    @dbus.service.signal(CHANNEL_TYPE_TEXT, signature='uuuuus')
    def Received(self, id, timestamp, sender, type, flags, text):
        self._pending_messages[id] = (timestamp, sender, type, flags, text)

from telepathy._generated.Channel_Interface_Chat_State \
        import ChannelInterfaceChatState

from telepathy._generated.Channel_Interface_DTMF import ChannelInterfaceDTMF

from telepathy._generated.Channel_Interface_Group \
        import ChannelInterfaceGroup as _ChannelInterfaceGroup

class ChannelInterfaceGroup(_ChannelInterfaceGroup, DBusProperties):

    def __init__(self):
        _ChannelInterfaceGroup.__init__(self)
        DBusProperties.__init__(self)

        self._implement_property_get(CHANNEL_INTERFACE_GROUP,
            {'GroupFlags': lambda: dbus.UInt32(self.GetGroupFlags()),
             'Members': lambda: dbus.Array(self.GetMembers(), signature='u'),
             'RemotePendingMembers': lambda: dbus.Array(self.GetRemotePendingMembers(), signature='u'),
             'SelfHandle': lambda: dbus.UInt32(self.GetSelfHandle())})

        self._group_flags = 0
        self._members = set()
        self._local_pending = set()
        self._remote_pending = set()

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='u')
    def GetGroupFlags(self):
        return self._group_flags

    @dbus.service.signal(CHANNEL_INTERFACE_GROUP, signature='uu')
    def GroupFlagsChanged(self, added, removed):
        self._group_flags |= added
        self._group_flags &= ~removed

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='au')
    def GetMembers(self):
        return self._members

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='u')
    def GetSelfHandle(self):
        self_handle = self._conn.GetSelfHandle()
        if (self_handle in self._members or
            self_handle in self._local_pending or
            self_handle in self._remote_pending):
            return self_handle
        else:
            return 0

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='au')
    def GetLocalPendingMembers(self):
        return self._local_pending

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='au')
    def GetRemotePendingMembers(self):
        return self._remote_pending

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='auauau')
    def GetAllMembers(self):
        return (self._members, self._local_pending, self._remote_pending)

    @dbus.service.signal(CHANNEL_INTERFACE_GROUP, signature='sauauauauuu')
    def MembersChanged(self, message, added, removed, local_pending, remote_pending, actor, reason):

        self._members.update(added)
        self._members.difference_update(removed)

        self._local_pending.update(local_pending)
        self._local_pending.difference_update(added)
        self._local_pending.difference_update(removed)

        self._remote_pending.update(remote_pending)
        self._remote_pending.difference_update(added)
        self._remote_pending.difference_update(removed)

from telepathy._generated.Channel_Interface_Hold import ChannelInterfaceHold

# ChannelInterfaceMediaSignalling is in telepathy.server.media

from telepathy._generated.Channel_Interface_Password \
        import ChannelInterfacePassword as _ChannelInterfacePassword

class ChannelInterfacePassword(_ChannelInterfacePassword):
    def __init__(self):
        _ChannelInterfacePassword.__init__(self)
        self._password_flags = 0
        self._password = ''

    @dbus.service.method(CHANNEL_INTERFACE_PASSWORD, in_signature='', out_signature='u')
    def GetPasswordFlags(self):
        return self._password_flags

    @dbus.service.signal(CHANNEL_INTERFACE_PASSWORD, signature='uu')
    def PasswordFlagsChanged(self, added, removed):
        self._password_flags |= added
        self._password_flags &= ~removed

from telepathy._generated.Channel_Interface_Call_State import ChannelInterfaceCallState
<commit_msg>channel: don't find the channel handle if TargetHandleType=NONE or TargetHandle is missing<commit_after># telepathy-python - Base classes defining the interfaces of the Telepathy framework
#
# Copyright (C) 2005, 2006 Collabora Limited
# Copyright (C) 2005, 2006 Nokia Corporation
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

import dbus
import dbus.service

from telepathy.constants import (CONNECTION_HANDLE_TYPE_NONE,
                                 CHANNEL_TEXT_MESSAGE_TYPE_NORMAL,
                                 HANDLE_TYPE_NONE)

from telepathy.errors import InvalidArgument

from telepathy.interfaces import (CHANNEL_INTERFACE,
                                  CHANNEL_INTERFACE_DTMF,
                                  CHANNEL_INTERFACE_GROUP,
                                  CHANNEL_INTERFACE_HOLD,
                                  CHANNEL_INTERFACE_PASSWORD,
                                  CHANNEL_TYPE_CONTACT_LIST,
                                  CHANNEL_TYPE_FILE_TRANSFER,
                                  CHANNEL_TYPE_ROOM_LIST,
                                  CHANNEL_TYPE_STREAMED_MEDIA,
                                  CHANNEL_TYPE_TEXT,
                                  MEDIA_SESSION_HANDLER,
                                  MEDIA_STREAM_HANDLER)

from telepathy._generated.Channel import Channel as _Channel

from telepathy.server.properties import DBusProperties

class Channel(_Channel, DBusProperties):

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the base channel object.

        Parameters:
        connection - the parent Connection object
        props - initial channel properties
        """
        self._conn = connection
        self._chan_manager = manager

        object_path = self._conn.get_channel_path(object_path)
        _Channel.__init__(self, self._conn._name, object_path)

        self._type = props[CHANNEL_INTERFACE + '.ChannelType']
        self._requested = props[CHANNEL_INTERFACE + '.Requested']

        self._immutable_properties = dict()

        if CHANNEL_INTERFACE + '.TargetHandleType' in props \
                and CHANNEL_INTERFACE + '.TargetHandle' in props:
            if props[CHANNEL_INTERFACE + '.TargetHandleType'] == HANDLE_TYPE_NONE:
                self._handle = None
            else:
                self._handle = self._conn.handle(
                    props[CHANNEL_INTERFACE + '.TargetHandleType'],
                    props[CHANNEL_INTERFACE + '.TargetHandle'])
        else:
            self._handle = None

        self._interfaces = set()

        DBusProperties.__init__(self)
        self._implement_property_get(CHANNEL_INTERFACE,
            {'ChannelType': lambda: dbus.String(self.GetChannelType()),
             'Interfaces': lambda: dbus.Array(self.GetInterfaces(), signature='s'),
             'TargetHandle': lambda: dbus.UInt32(self._get_target_handle()),
             'TargetHandleType': lambda: dbus.UInt32(self._get_handle_type()),
             'TargetID': lambda: dbus.String(self._get_target_id()),
             'Requested': lambda: self._requested})

        self._add_immutables({
            'ChannelType': CHANNEL_INTERFACE,
            'TargetHandle': CHANNEL_INTERFACE,
            'Interfaces': CHANNEL_INTERFACE,
            'TargetHandleType': CHANNEL_INTERFACE,
            'TargetID': CHANNEL_INTERFACE,
            'Requested': CHANNEL_INTERFACE
            })

    def _add_immutables(self, props):
        self._immutable_properties.update(props)

    def _get_target_handle(self):
        if self._handle:
            return self._handle.get_id()
        else:
            return 0

    def _get_handle_type(self):
        if self._handle:
            return self._handle.get_type()
        else:
            return CONNECTION_HANDLE_TYPE_NONE

    def _get_target_id(self):
        if self._handle:
            return self._handle.get_name()
        else:
            return ''

    def get_props(self):
        props = dict()
        for prop, iface in self._immutable_properties.items():
            props[iface + '.' + prop] = \
                self._prop_getters[iface][prop]()
        return props

    @dbus.service.method(CHANNEL_INTERFACE, in_signature='', out_signature='')
    def Close(self):
        self.Closed()
        self._chan_manager.remove_channel(self)
        self._conn.remove_channel(self)

    @dbus.service.method(CHANNEL_INTERFACE, in_signature='', out_signature='s')
    def GetChannelType(self):
        """ Returns the interface name for the type of this channel. """
        return self._type

    @dbus.service.method(CHANNEL_INTERFACE, in_signature='', out_signature='uu')
    def GetHandle(self):
        """ Returns the handle type and number if this channel represents a
        communication with a particular contact, room or server-stored list, or
        zero if it is transient and defined only by its contents. """
        if self._handle:
            return self._handle.get_type(), self._handle
        else:
            return (CONNECTION_HANDLE_TYPE_NONE, 0)

    @dbus.service.method(CHANNEL_INTERFACE, in_signature='', out_signature='as')
    def GetInterfaces(self):
        """
        Get the optional interfaces implemented by the channel.

        Returns:
        an array of the D-Bus interface names
        """
        return self._interfaces

from telepathy._generated.Channel_Type_Contact_List \
        import ChannelTypeContactList as _ChannelTypeContactListIface

class ChannelTypeContactList(Channel, _ChannelTypeContactListIface):
    __doc__ = _ChannelTypeContactListIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)

from telepathy._generated.Channel_Type_File_Transfer \
        import ChannelTypeFileTransfer as _ChannelTypeFileTransferIface

class ChannelTypeFileTransfer(Channel, _ChannelTypeFileTransferIface):
    __doc__ = _ChannelTypeFileTransferIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)

from telepathy._generated.Channel_Type_Streamed_Media \
        import ChannelTypeStreamedMedia as _ChannelTypeStreamedMediaIface

class ChannelTypeStreamedMedia(Channel, _ChannelTypeStreamedMediaIface):
    __doc__ = _ChannelTypeStreamedMediaIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)

from telepathy._generated.Channel_Type_Room_List \
        import ChannelTypeRoomList as _ChannelTypeRoomListIface

class ChannelTypeRoomList(Channel, _ChannelTypeRoomListIface):
    __doc__ = _ChannelTypeRoomListIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)
        self._listing_rooms = False
        self._rooms = {}

        self._add_immutables(self, {'Server': CHANNEL_TYPE_ROOM_LIST})

    @dbus.service.method(CHANNEL_TYPE_ROOM_LIST, in_signature='', out_signature='b')
    def GetListingRooms(self):
        return self._listing_rooms

    @dbus.service.signal(CHANNEL_TYPE_ROOM_LIST, signature='b')
    def ListingRooms(self, listing):
        self._listing_rooms = listing

from telepathy._generated.Channel_Type_Text \
        import ChannelTypeText as _ChannelTypeTextIface

class ChannelTypeText(Channel, _ChannelTypeTextIface):
    __doc__ = _ChannelTypeTextIface.__doc__

    def __init__(self, connection, manager, props, object_path=None):
        """
        Initialise the channel.

        Parameters:
        connection - the parent Telepathy Connection object
        """
        Channel.__init__(self, connection, manager, props,
            object_path=object_path)

        self._pending_messages = {}
        self._message_types = [CHANNEL_TEXT_MESSAGE_TYPE_NORMAL]

    @dbus.service.method(CHANNEL_TYPE_TEXT, in_signature='', out_signature='au')
    def GetMessageTypes(self):
        """
        Return an array indicating which types of message may be sent on this
        channel.

        Returns:
        an array of integer message types as defined above
        """
        return self._message_types

    @dbus.service.method(CHANNEL_TYPE_TEXT, in_signature='au', out_signature='')
    def AcknowledgePendingMessages(self, ids):
        """
        Inform the channel that you have handled messages by displaying them to
        the user (or equivalent), so they can be removed from the pending queue.

        Parameters:
        ids - the message to acknowledge

        Possible Errors:
        InvalidArgument (a given message ID was not found, no action taken)
        """
        for id in ids:
            if id not in self._pending_messages:
                raise InvalidArgument("the given message ID was not found")

        for id in ids:
            del self._pending_messages[id]

    @dbus.service.method(CHANNEL_TYPE_TEXT, in_signature='b', out_signature='a(uuuuus)')
    def ListPendingMessages(self, clear):
        """
        List the messages currently in the pending queue, and optionally
        remove then all.

        Parameters:
        clear - a boolean indicating whether the queue should be cleared

        Returns:
        an array of structs containing:
            a numeric identifier
            a unix timestamp indicating when the message was received
            an integer handle of the contact who sent the message
            an integer of the message type
            a bitwise OR of the message flags
            a string of the text of the message
        """
        messages = []
        for id in self._pending_messages.keys():
            (timestamp, sender, type, flags, text) = self._pending_messages[id]
            message = (id, timestamp, sender, type, flags, text)
            messages.append(message)
            if clear:
                del self._pending_messages[id]
        messages.sort(cmp=lambda x,y:cmp(x[1], y[1]))
        return messages

    @dbus.service.signal(CHANNEL_TYPE_TEXT, signature='uuuuus')
    def Received(self, id, timestamp, sender, type, flags, text):
        self._pending_messages[id] = (timestamp, sender, type, flags, text)

from telepathy._generated.Channel_Interface_Chat_State \
        import ChannelInterfaceChatState

from telepathy._generated.Channel_Interface_DTMF import ChannelInterfaceDTMF

from telepathy._generated.Channel_Interface_Group \
        import ChannelInterfaceGroup as _ChannelInterfaceGroup

class ChannelInterfaceGroup(_ChannelInterfaceGroup, DBusProperties):

    def __init__(self):
        _ChannelInterfaceGroup.__init__(self)
        DBusProperties.__init__(self)

        self._implement_property_get(CHANNEL_INTERFACE_GROUP,
            {'GroupFlags': lambda: dbus.UInt32(self.GetGroupFlags()),
             'Members': lambda: dbus.Array(self.GetMembers(), signature='u'),
             'RemotePendingMembers': lambda: dbus.Array(self.GetRemotePendingMembers(), signature='u'),
             'SelfHandle': lambda: dbus.UInt32(self.GetSelfHandle())})

        self._group_flags = 0
        self._members = set()
        self._local_pending = set()
        self._remote_pending = set()

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='u')
    def GetGroupFlags(self):
        return self._group_flags

    @dbus.service.signal(CHANNEL_INTERFACE_GROUP, signature='uu')
    def GroupFlagsChanged(self, added, removed):
        self._group_flags |= added
        self._group_flags &= ~removed

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='au')
    def GetMembers(self):
        return self._members

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='u')
    def GetSelfHandle(self):
        self_handle = self._conn.GetSelfHandle()
        if (self_handle in self._members or
            self_handle in self._local_pending or
            self_handle in self._remote_pending):
            return self_handle
        else:
            return 0

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='au')
    def GetLocalPendingMembers(self):
        return self._local_pending

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='au')
    def GetRemotePendingMembers(self):
        return self._remote_pending

    @dbus.service.method(CHANNEL_INTERFACE_GROUP, in_signature='', out_signature='auauau')
    def GetAllMembers(self):
        return (self._members, self._local_pending, self._remote_pending)

    @dbus.service.signal(CHANNEL_INTERFACE_GROUP, signature='sauauauauuu')
    def MembersChanged(self, message, added, removed, local_pending, remote_pending, actor, reason):

        self._members.update(added)
        self._members.difference_update(removed)

        self._local_pending.update(local_pending)
        self._local_pending.difference_update(added)
        self._local_pending.difference_update(removed)

        self._remote_pending.update(remote_pending)
        self._remote_pending.difference_update(added)
        self._remote_pending.difference_update(removed)

from telepathy._generated.Channel_Interface_Hold import ChannelInterfaceHold

# ChannelInterfaceMediaSignalling is in telepathy.server.media

from telepathy._generated.Channel_Interface_Password \
        import ChannelInterfacePassword as _ChannelInterfacePassword

class ChannelInterfacePassword(_ChannelInterfacePassword):
    def __init__(self):
        _ChannelInterfacePassword.__init__(self)
        self._password_flags = 0
        self._password = ''

    @dbus.service.method(CHANNEL_INTERFACE_PASSWORD, in_signature='', out_signature='u')
    def GetPasswordFlags(self):
        return self._password_flags

    @dbus.service.signal(CHANNEL_INTERFACE_PASSWORD, signature='uu')
    def PasswordFlagsChanged(self, added, removed):
        self._password_flags |= added
        self._password_flags &= ~removed

from telepathy._generated.Channel_Interface_Call_State import ChannelInterfaceCallState
<|endoftext|>


<commit_before>#--
# Copyright (c) 2008-2011 Net-ng.
# All rights reserved.
#
# This software is licensed under the BSD License, as described in
# the file LICENSE.txt, which you should have received as part of
# this distribution.
#--

"""Simple form based authentication manager

The id and password of the user are first searched into the parameters of
the request. So, first, set a form with the fields names ``__ac_name``
and ``__ac_password`` (the prefix ``__ac`` is configurable).

Then the user id and the password are automatically kept into a cookie,
sent back on each request by the browser.

.. warning::

   This simple authentication manager keeps the user id and password in
   clear into the cookie. So this authentication manager is as secure as
   the HTTP basic authentication.
"""
import webob

from nagare import security
from nagare.security import basic_auth

class Authentication(basic_auth.Authentication):
    """Simple form based authentication"""

    def __init__(self, prefix='__ac', key=None, max_age=None,
                 path='/', domain=None, secure=None, httponly=False,
                 version=None, comment=None, expires=None,
                 realm=None):
        """Initialization

        In:
          - ``prefix`` -- prefix of the names of the user id and password fields
            into the form
          - ``realm`` -- is the form based authentication completed by a
            basic HTTP authentication ?
          - all the other keyword parameters are passed to the ``set_cookie()``
            method of the ``WebOb`` response object
            (see http://pythonpaste.org/webob/reference.html#id5)
        """
        super(Authentication, self).__init__(realm)
        self.prefix = prefix

        self.key = key or prefix
        self.max_age = max_age
        self.path = path
        self.domain = domain
        self.secure = secure
        self.httponly = httponly
        self.version = version
        self.comment = comment
        self.expires = expires

    def get_ids_from_params(self, params):
        """Search the data associated with the connected user into the request
        parameter

        In:
          - ``params`` -- the request parameters

        Return:
          - A tuple with the id of the user and its password
        """
        return (params.get(self.prefix+'_name'), params.get(self.prefix+'_password'))

    def cookie_decode(self, cookie):
        """Decode the data of the user cookie

        In:
          - ``cookie`` -- the data of the user cookie

        Return:
          - A list with the id of the user and its password
        """
        return [s.decode('base64').decode('utf-8') for s in cookie.split(':')]

    def get_ids_from_cookie(self, cookies):
        """Search the data associated with the connected user into the cookies

        In:
          - ``cookies`` -- cookies dictionary

        Return:
          - A list with the id of the user and its password
        """
        data = cookies.get(self.key)
        if not data:
            return (None, None)

        return self.cookie_decode(data)

    def cookie_encode(self, username, password):
        """Encode the data of the user cookie

        In:
          - ``username`` -- name (login) of the user
          - ``password`` -- password of the user

        Return:
          - the data to put into the user cookie
        """
        return '%s:%s' % (
                          username.encode('utf-8').encode('base64').rstrip(),
                          password.encode('utf-8').encode('base64').rstrip()
                         )

    def _get_ids(self, request, response):
        """Return the data associated with the connected user

        In:
          - ``request`` -- the web request object
          - ``response`` -- the web response object

        Return:
          - A list with the id of the user and its password
        """
        # First, search into the request parameters
        ids = self.get_ids_from_params(request.params)
        if not all(ids):
            # Second, search into the cookies
            ids = self.get_ids_from_cookie(request.cookies)
            if not all(ids) and self.realm:
                # Third, if a realm is set, look into the basic authentication header
                ids = super(Authentication, self)._get_ids(request, response)

        return ids

    def set_user_id(self, user, id, password):
        """Set the credentials of the user

        In:
          - ``user`` -- the user
          - ``id`` -- the user id
          - ``password`` -- the user password
        """
        user.set_id(id, password)

    def end_rendering(self, request, response, session):
        """End of the request processing

        In:
          - ``request`` -- the request object
          - ``response`` -- the response object
          - ``session`` -- the session
        """
        user = security._get_user()

        if user:
            if not user.expired:
                response.set_cookie(
                                self.key, self.cookie_encode(*user.get_id()),
                                max_age=self.max_age,
                                path=self.path,
                                domain=self.domain,
                                secure=self.secure,
                                httponly=self.httponly,
                                version=self.version,
                                comment=self.comment,
                                expires=self.expires
                            )
            else:
                response.delete_cookie(self.key, self.path, self.domain)

                if user.delete_session:
                    session.delete()

                location = user.logout_location
                if location is not None:
                    if not location.startswith(('http', '/')):
                        location = request.application_url + '/' + location

                    response.status = 301
                    response.location = location
                    response.body = ''

    def denies(self, detail):
        """Method called when a permission is denied

        In:
          - ``details`` -- a ``security.common.denial`` object
        """
        if self.realm:
            super(Authentication, self).denies(detail)

        raise webob.exc.HTTPForbidden(str(detail))

    def logout(self, location='', delete_session=True):
        """Deconnection of the current user

        Mark the user object as expired

        In:
          - ``location`` -- location to redirect to
          - ``delete_session`` -- is the session expired too ?
        """
        user = security.get_user()

        user.logout_location = location
        user.delete_session = delete_session
        user.expired = True
<commit_msg>New WebOb `Response.set_cookie()` signature<commit_after>#--
# Copyright (c) 2008-2011 Net-ng.
# All rights reserved.
#
# This software is licensed under the BSD License, as described in
# the file LICENSE.txt, which you should have received as part of
# this distribution.
#--

"""Simple form based authentication manager

The id and password of the user are first searched into the parameters of
the request. So, first, set a form with the fields names ``__ac_name``
and ``__ac_password`` (the prefix ``__ac`` is configurable).

Then the user id and the password are automatically kept into a cookie,
sent back on each request by the browser.

.. warning::

   This simple authentication manager keeps the user id and password in
   clear into the cookie. So this authentication manager is as secure as
   the HTTP basic authentication.
"""
import webob

from nagare import security
from nagare.security import basic_auth

class Authentication(basic_auth.Authentication):
    """Simple form based authentication"""

    def __init__(self, prefix='__ac', key=None, max_age=None,
                 path='/', domain=None, secure=None, httponly=False,
                 comment=None, expires=None, overwrite=False,
                 realm=None):
        """Initialization

        In:
          - ``prefix`` -- prefix of the names of the user id and password fields
            into the form
          - ``realm`` -- is the form based authentication completed by a
            basic HTTP authentication ?
          - all the other keyword parameters are passed to the ``set_cookie()``
            method of the ``WebOb`` response object
            (see http://pythonpaste.org/webob/reference.html#id5)
        """
        super(Authentication, self).__init__(realm)
        self.prefix = prefix

        self.key = key or prefix
        self.max_age = max_age
        self.path = path
        self.domain = domain
        self.secure = secure
        self.httponly = httponly
        self.comment = comment
        self.expires = expires
        self.overwrite = overwrite

    def get_ids_from_params(self, params):
        """Search the data associated with the connected user into the request
        parameter

        In:
          - ``params`` -- the request parameters

        Return:
          - A tuple with the id of the user and its password
        """
        return (params.get(self.prefix+'_name'), params.get(self.prefix+'_password'))

    def cookie_decode(self, cookie):
        """Decode the data of the user cookie

        In:
          - ``cookie`` -- the data of the user cookie

        Return:
          - A list with the id of the user and its password
        """
        return [s.decode('base64').decode('utf-8') for s in cookie.split(':')]

    def get_ids_from_cookie(self, cookies):
        """Search the data associated with the connected user into the cookies

        In:
          - ``cookies`` -- cookies dictionary

        Return:
          - A list with the id of the user and its password
        """
        data = cookies.get(self.key)
        if not data:
            return (None, None)

        return self.cookie_decode(data)

    def cookie_encode(self, username, password):
        """Encode the data of the user cookie

        In:
          - ``username`` -- name (login) of the user
          - ``password`` -- password of the user

        Return:
          - the data to put into the user cookie
        """
        return '%s:%s' % (
                          username.encode('utf-8').encode('base64').rstrip(),
                          password.encode('utf-8').encode('base64').rstrip()
                         )

    def _get_ids(self, request, response):
        """Return the data associated with the connected user

        In:
          - ``request`` -- the web request object
          - ``response`` -- the web response object

        Return:
          - A list with the id of the user and its password
        """
        # First, search into the request parameters
        ids = self.get_ids_from_params(request.params)
        if not all(ids):
            # Second, search into the cookies
            ids = self.get_ids_from_cookie(request.cookies)
            if not all(ids) and self.realm:
                # Third, if a realm is set, look into the basic authentication header
                ids = super(Authentication, self)._get_ids(request, response)

        return ids

    def set_user_id(self, user, id, password):
        """Set the credentials of the user

        In:
          - ``user`` -- the user
          - ``id`` -- the user id
          - ``password`` -- the user password
        """
        user.set_id(id, password)

    def end_rendering(self, request, response, session):
        """End of the request processing

        In:
          - ``request`` -- the request object
          - ``response`` -- the response object
          - ``session`` -- the session
        """
        user = security._get_user()

        if user:
            if not user.expired:
                response.set_cookie(
                                self.key, self.cookie_encode(*user.get_id()),
                                max_age=self.max_age,
                                path=self.path,
                                domain=self.domain,
                                secure=self.secure,
                                httponly=self.httponly,
                                comment=self.comment,
                                expires=self.expires,
                                overwrite=self.overwrite
                            )
            else:
                response.delete_cookie(self.key, self.path, self.domain)

                if user.delete_session:
                    session.delete()

                location = user.logout_location
                if location is not None:
                    if not location.startswith(('http', '/')):
                        location = request.application_url + '/' + location

                    response.status = 301
                    response.location = location
                    response.body = ''

    def denies(self, detail):
        """Method called when a permission is denied

        In:
          - ``details`` -- a ``security.common.denial`` object
        """
        if self.realm:
            super(Authentication, self).denies(detail)

        raise webob.exc.HTTPForbidden(str(detail))

    def logout(self, location='', delete_session=True):
        """Deconnection of the current user

        Mark the user object as expired

        In:
          - ``location`` -- location to redirect to
          - ``delete_session`` -- is the session expired too ?
        """
        user = security.get_user()

        user.logout_location = location
        user.delete_session = delete_session
        user.expired = True
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
# The MIT License
#
# Copyright (c) 2016 Grigory Chernyshev
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from StringIO import StringIO
import csv

class PropertyManager(object):
    def __init__(
        self,
        session,
        pipeline_name=None,
        pipeline_counter=None,
        stage_name=None,
        stage_counter=None,
        job_name=None
    ):
        """
        :type session: yagocd.session.Session
        """
        self._session = session
        self.base_api = self._session.base_api(api_path='')

        self._pipeline_name = pipeline_name
        self._pipeline_counter = pipeline_counter
        self._stage_name = stage_name
        self._stage_counter = stage_counter
        self._job_name = job_name

    def list(
        self,
        pipeline_name=None,
        pipeline_counter=None,
        stage_name=None,
        stage_counter=None,
        job_name=None
    ):
        assert self._pipeline_name or pipeline_name
        assert self._pipeline_counter or pipeline_counter
        assert self._stage_name or stage_name
        assert self._stage_counter or stage_counter
        assert self._job_name or job_name

        response = self._session.post(
            path='{base_api}/properties/{pipeline_name}/{pipeline_counter}/{stage_name}/{stage_counter}/{job_name}'.format(
                base_api=self.base_api,
                pipeline_name=self._pipeline_name or pipeline_name,
                pipeline_counter=self._pipeline_counter or pipeline_counter,
                stage_name=self._stage_name or stage_name,
                stage_counter=self._stage_counter or stage_counter,
                job_name=self._job_name or job_name
            ),
            headers={'Accept': 'application/json'},
        )
        text = StringIO(response.text)
        parsed = list(csv.reader(text))
        properties = dict(zip(parsed[0], parsed[1]))

        return properties

if __name__ == '__main__':
    pass
<commit_msg>Add method for getting single property.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
# The MIT License
#
# Copyright (c) 2016 Grigory Chernyshev
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from StringIO import StringIO
import csv

class PropertyManager(object):
    def __init__(
        self,
        session,
        pipeline_name=None,
        pipeline_counter=None,
        stage_name=None,
        stage_counter=None,
        job_name=None
    ):
        """
        :type session: yagocd.session.Session
        """
        self._session = session
        self.base_api = self._session.base_api(api_path='')

        self._pipeline_name = pipeline_name
        self._pipeline_counter = pipeline_counter
        self._stage_name = stage_name
        self._stage_counter = stage_counter
        self._job_name = job_name

    def list(
        self,
        pipeline_name=None,
        pipeline_counter=None,
        stage_name=None,
        stage_counter=None,
        job_name=None
    ):
        assert self._pipeline_name or pipeline_name
        assert self._pipeline_counter or pipeline_counter
        assert self._stage_name or stage_name
        assert self._stage_counter or stage_counter
        assert self._job_name or job_name

        response = self._session.post(
            path='{base_api}/properties/{pipeline_name}/{pipeline_counter}/{stage_name}/{stage_counter}/{job_name}'.format(
                base_api=self.base_api,
                pipeline_name=self._pipeline_name or pipeline_name,
                pipeline_counter=self._pipeline_counter or pipeline_counter,
                stage_name=self._stage_name or stage_name,
                stage_counter=self._stage_counter or stage_counter,
                job_name=self._job_name or job_name
            ),
            headers={'Accept': 'application/json'},
        )
        text = StringIO(response.text)
        parsed = list(csv.reader(text))
        properties = dict(zip(parsed[0], parsed[1]))

        return properties

    def get(
        self,
        name,
        pipeline_name=None,
        pipeline_counter=None,
        stage_name=None,
        stage_counter=None,
        job_name=None
    ):
        assert self._pipeline_name or pipeline_name
        assert self._pipeline_counter or pipeline_counter
        assert self._stage_name or stage_name
        assert self._stage_counter or stage_counter
        assert self._job_name or job_name

        response = self._session.post(
            path='{base_api}/properties/{pipeline_name}/{pipeline_counter}/{stage_name}/{stage_counter}/{job_name}/{name}'.format(
                base_api=self.base_api,
                pipeline_name=self._pipeline_name or pipeline_name,
                pipeline_counter=self._pipeline_counter or pipeline_counter,
                stage_name=self._stage_name or stage_name,
                stage_counter=self._stage_counter or stage_counter,
                job_name=self._job_name or job_name,
                name=name
            ),
            headers={'Accept': 'application/json'},
        )
        text = StringIO(response.text)
        parsed = list(csv.reader(text))
        properties = dict(zip(parsed[0], parsed[1]))

        return properties

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
import xdelta3
import os
import commands
import filecmp
import sys
import subprocess
import getopt
import time
import socket
from datetime import datetime, timedelta
import telnetlib
import pylzma
from flask import Flask,flash, request,render_template, Response,session,g
from flask import Flask, request, session, g, redirect, url_for, abort, render_template, flash, Response
import re
import json
from cloudlet import telnet_connection_waiting

# global constant and variable
WEB_SERVER_PORT_NUMBER = 9091
ISR_ORIGIN_SRC_PATH = '/home/krha/cloudlet/src/ISR/src'
ISR_ANDROID_SRC_PATH = '/home/krha/cloudlet/src/ISR/src-mock'
ISR_ANDROID_PARCEL_PATH = '/home/krha/cloudlet/src/ISR/parcel'
VNC_PATH = '/home/krha/.isr/'
user_name = ''
server_address = ''
launch_start = datetime.now()
launch_end = datetime.now()
application_names = ("moped", "face", "null", "graphics", "speech")

# web server configuration
app = Flask(__name__)
app.config.from_object(__name__)

# web server for receiving command
@app.route("/isr", methods=['POST'])
def isr():
    global user_name
    global server_address

    print "Receive isr_info (run-type, application name) from client"
    json_data = request.form["info"]
    metadata = json.loads(json_data)

    run_type = metadata['run-type'].lower()
    application_name = metadata['application'].lower()

    if run_type in ("cloud", "mobile") and application_names:
        # Run application
        if run_type == "cloud":
            print "Client request : %s, %s --> connecting to %s with %s" % (run_type, application_name, server_address, user_name)
            ret = do_cloud_isr(user_name, application_name, server_address)
        elif run_type == "mobile":
            print "Client request : %s, %s --> connecting to %s with %s" % (run_type, application_name, server_address, user_name)
            ret = do_mobile_isr(user_name, application_name, server_address)

        if ret:
            print "SUCCESS"
            return "SUCCESS"

    ret_msg = "Wrong parameter " + run_type + ", " + application_name
    print ret_msg
    return ret_msg

def recompile_isr(src_path):
    command_str = 'cd %s && sudo make && sudo make install' % (src_path)
    print command_str
    ret1, ret_string = commands.getstatusoutput(command_str)
    if ret1 != 0:
        raise "Cannot compile ISR"
    return True

# command Login
def login(user_name, server_address):
    command_str = 'isr auth -s ' + server_address + ' -u ' + user_name
    ret, ret_string = commands.getstatusoutput(command_str)

    if ret == 0:
        return True, ''
    return False, "Cannot connected to Server %s, %s" % (server_address, ret_string)

def get_uuid(user_name, server_address, vm_name):
    # list cache
    command_str = 'isr lshoard -l -s ' + server_address + ' -u ' + user_name
    print command_str
    ret, ret_string = commands.getstatusoutput(command_str)
    if ret != 0:
        return False, "Cannot list up VM hoard"

    # find UUID, which has vm_name
    lines = ret_string.split('\n')
    uuid = None
    for index, line in enumerate(lines):
        if line.find(vm_name) != -1 and len(lines) > (index+1):
            uuid = lines[index+1].lstrip().split(" ")[0]

    return uuid

# remove all cache
def remove_cache(user_name, server_address, vm_name):
    uuid = get_uuid(user_name, server_address, vm_name)
    if uuid != None:
        # erase disk cache
        command_str = 'isr rmhoard ' + uuid + ' -s ' + server_address + ' -u ' + user_name
        ret, ret_string = commands.getstatusoutput(command_str)

    # erase memory
    '''
    mem_dir = '/home/krha/.isr/hoard/img/'
    print 'aaaaaaaaaaaaaaaaaaaaaaaaaa'
    if os.path.exists(mem_dir):
        print 'aaaaaaaaaaaaaaaaaaaaaaaaaa'
        command_str = "rm -rf %s*" %(mem_dir)
        print command_str
        ret, ret_string = commands.getstatusoutput(command_str)
    '''

    mem_dir = '/home/krha/.isr/'
    if os.path.exists(mem_dir):
        command_str = "rm -rf %s" % (mem_dir)
        print command_str
        ret, ret_string = commands.getstatusoutput(command_str)

    return True, ''

# resume VM
def resume_vm(user_name, server_address, vm_name):
    time_start = datetime.now()
    time_end = datetime.now()
    time_transfer_start = datetime.now()
    time_transfer_end = datetime.now()
    time_decomp_mem_start = datetime.now()
    time_decomp_mem_end = datetime.now()
    time_kvm_start = datetime.now()
    time_kvm_end = datetime.now()

    command_str = 'isr resume ' + vm_name + ' -s ' + server_address + ' -u ' + user_name + ' -F -D'
    print command_str
    time_start = datetime.now()
    print "VM Resume Process start : " + str(time_start)
    proc = subprocess.Popen(command_str, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    while True:
        time.sleep(0.1)
        output = proc.stdout.readline()
        if len(output.strip()) != 0 and output.find("[krha]") == -1:
            sys.stdout.write(output)

        # time stamping using log from isr_client
        # Not reliable but fast for simple test
        if output.strip().find("Fetching keyring") == 0:
            time_transfer_start = datetime.now()
        elif output.strip().find("Decrypting and uncompressing") == 0:
            time_transfer_end = datetime.now()
            time_decomp_mem_start = datetime.now()
        elif output.strip().find("Updating hoard cache") == 0:
            time_decomp_mem_end = datetime.now()
        elif output.strip().find("Launching KVM") == 0:
            time_kvm_start = datetime.now()
            break;

    # waiting for TCP socket open
    # predefined for test, it is opened at ISR/vmm/kvm
    # So, no multiple ISR Client at one machine
    telnet_port = 9998
    for i in xrange(200):
        command_str = "netstat -an | grep 127.0.0.1:" + str(telnet_port)
        proc = subprocess.Popen(command_str, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        output = proc.stdout.readline()
        if output.find("LISTEN") != -1:
            break;
        time.sleep(0.1)

    # Getting VM Status information through Telnet
    telnet_connection_waiting(telnet_port)
    time_kvm_end = datetime.now()

    '''
    # Waiting for kvm.vnc
    # find UUID, which has vm_name
    uuid = get_uuid(user_name, server_address, vm_name)
    vnc_path = os.path.join(VNC_PATH, uuid, "cfg", "kvm.vnc")
    for i in xrange(200):
        if os.path.exists(vnc_path):
            time_kvm_end = datetime.now()
            print "VM Resume Process End : " + str(time_kvm_end)
            break;
        time.sleep(0.1)
    '''

    # if we wait for process to end, we cannot return to web client
    # ret = proc.wait()
    time_end = datetime.now()
    print "Return from Resume"
    print "[Total Time] : ", str(time_end-time_start)
    print '[Transfer Time(Memory)] : ', str(time_transfer_end-time_transfer_start)
    print '[Decompression Time] : ', str(time_decomp_mem_end-time_decomp_mem_start)
    print '[KVM Time] : ', str(time_kvm_end-time_kvm_start)

# stop VM
def stop_vm(user_name, server_address, vm_name):
    command_str = 'isr clean ' + vm_name + ' -s ' + server_address + ' -u ' + user_name + ' -f'
    print command_str
    proc = subprocess.Popen(command_str, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    proc.stdin.write('y\n')

    return True

# Exit with error message
def exit_error(error_message):
    print 'Error, ', error_message
    sys.exit(1)

def do_cloud_isr(user_name, vm_name, server_address):
    # compile ISR again, because we have multiple version of ISR such as mock android
    # This is not good approach, but easy for simple test
    # I'll gonna erase this script after submission :(
    recompile_isr(ISR_ORIGIN_SRC_PATH)

    # step1. login
    ret, err = login(user_name, server_address)
    if not ret:
        print err
        return False

    # step2. remove all cache
    ret, err = remove_cache(user_name, server_address, vm_name)
    if not ret:
        return False

    # step3. resume VM, wait until finish (close window)
    start_time = datetime.now()
    resume_vm(user_name, server_address, vm_name)
    end_time = datetime.now()

    return True

def do_mobile_isr(user_name, vm_name, server_address):
    # compile ISR again, because we have multiple version of ISR such as mock android
    # This is not good approach, but easy for simple test
    # I'll gonna erase this script after submission :(
    recompile_isr(ISR_ANDROID_SRC_PATH)

    # Change parcel to indicate it to mobile address, which is specified at server_address
    trick_parcel_address(ISR_ANDROID_PARCEL_PATH, vm_name, server_address)

    # step1. remove all cache
    ret, err = remove_cache(user_name, server_address, vm_name)
    if not ret:
        return False

    # step2. resume VM, wait until finish (close window)
    start_time = datetime.now()
    resume_vm(user_name, server_address, vm_name)
    end_time = datetime.now()
    return True

def trick_parcel_address(parcel_dir, vm_name, server_address):
    parcel_path = os.path.join(parcel_dir, vm_name, 'parcel.cfg')
    print parcel_path
    if not os.path.exists(parcel_path):
        print "Error, check you parcel file location : " + parcel_path
        sys.exit(2)
        return False
    lines = []
    fr = open(parcel_path, 'r')
    for line in fr:
        key = line.split("=")[0].strip()
        if key == "RPATH":
            lines.append("RPATH = http://" + server_address + ":8080\n")
        elif key == "SERVER":
            lines.append("SERVER = " + server_address + "\n")
        else:
            lines.append(line)
    fr.close()
    fw = open(parcel_path, 'w')
    fw.write(''.join(lines))
    fw.close()

def isr_clean_all(server_address, user_name):
    global application_names

    # kill all process that has 'isr'
    # I really hate this way, but it is almost only way that I can clean cache
    command_str = 'ps aux | grep isr'
    ret1, ret_string = commands.getstatusoutput(command_str)
    for line in ret_string.split('\n'):
        if line.find('isr') != -1 and line.find('isr_run.py') == -1 and line.find('vi ') == -1:
            pid = re.search('[A-Za-z]+\s+(\d+).*', line).groups(0)[0]
            command_str = 'kill -9 ' + pid
            print 'kill /isr + \t', command_str
            commands.getoutput(command_str)

    for vm_name in application_names:
        ret = stop_vm(user_name, server_address, vm_name)
        ret = remove_cache(user_name, server_address, vm_name)

def print_usage(program_name):
    print 'usage\t: %s [run|clean] [-u username] [-s server_address] ' % program_name
    print 'example\t: isr_run.py run -u cloudlet -s dagama.isr.cs.cmu.edu'

def main(argv):
    global user_name
    global server_address

    if len(argv) < 3:
        print_usage(os.path.basename(argv[0]))
        sys.exit(2)

    operation = argv[1].lower()
    if not operation in ("clean", "run"):
        print "No supporing operation : ", operation
        print_usage(os.path.basename(argv[0]))
        sys.exit(2)

    try:
        optlist, args = getopt.getopt(argv[2:], 'hu:s:', ["help", "user", "server"])
    except getopt.GetoptError, err:
        print str(err)
        print_usage(os.path.basename(argv[0]))
        sys.exit(2)

    # required input variables
    user_name = None
    server_address = None

    # parse argument
    for o, a in optlist:
        if o in ("-h", "--help"):
            print_usage(os.path.basename(argv[0]))
            sys.exit(0)
        elif o in ("-u", "--user"):
            user_name = a
        elif o in ("-s", "--server"):
            server_address = a
        else:
            assert False, "unhandled option"

    if user_name == None or server_address == None:
        print_usage(os.path.basename(argv[0]))
        sys.exit(2)

    # Always strat from the clean state
    isr_clean_all(server_address, user_name)
    if operation == "clean":
        sys.exit(0)

if __name__ == "__main__":
    main(sys.argv)
    #do_mobile_isr(user_name, "moped", server_address)
    app.run(host='0.0.0.0', port=WEB_SERVER_PORT_NUMBER, processes = 10)

<commit_msg>Test works with graphics<commit_after>#!/usr/bin/env python
import xdelta3
import os
import commands
import filecmp
import sys
import subprocess
import getopt
import time
import socket
from datetime import datetime, timedelta
import telnetlib
import pylzma
from flask import Flask,flash, request,render_template, Response,session,g
from flask import Flask, request, session, g, redirect, url_for, abort, render_template, flash, Response
import re
import json
from cloudlet import telnet_connection_waiting

# global constant and variable
WEB_SERVER_PORT_NUMBER = 9091
ISR_ORIGIN_SRC_PATH = '/home/krha/cloudlet/src/ISR/src-pipe'
ISR_ANDROID_SRC_PATH = '/home/krha/cloudlet/src/ISR/src-mock'
ISR_ANDROID_PARCEL_PATH = '/home/krha/cloudlet/src/ISR/parcel'
VNC_PATH = '/home/krha/.isr/'
user_name = ''
server_address = ''
launch_start = datetime.now()
launch_end = datetime.now()
application_names = ("moped", "face", "null", "graphics", "speech")

# web server configuration
app = Flask(__name__)
app.config.from_object(__name__)

# web server for receiving command
@app.route("/isr", methods=['POST'])
def isr():
    global user_name
    global server_address

    print "Receive isr_info (run-type, application name) from client"
    json_data = request.form["info"]
    metadata = json.loads(json_data)

    run_type = metadata['run-type'].lower()
    application_name = metadata['application'].lower()

    if run_type in ("cloud", "mobile") and application_names:
        # Run application
        if run_type == "cloud":
            print "Client request : %s, %s --> connecting to %s with %s" % (run_type, application_name, server_address, user_name)
            ret = do_cloud_isr(user_name, application_name, server_address)
        elif run_type == "mobile":
            print "Client request : %s, %s --> connecting to %s with %s" % (run_type, application_name, server_address, user_name)
            ret = do_mobile_isr(user_name, application_name, server_address)

        if ret:
            print "SUCCESS"
            return "SUCCESS"

    ret_msg = "Wrong parameter " + run_type + ", " + application_name
    print ret_msg
    return ret_msg

def recompile_isr(src_path):
    command_str = 'cd %s && sudo make && sudo make install' % (src_path)
    print command_str
    ret1, ret_string = commands.getstatusoutput(command_str)
    if ret1 != 0:
        raise "Cannot compile ISR"
    return True

# command Login
def login(user_name, server_address):
    command_str = 'isr auth -s ' + server_address + ' -u ' + user_name
    ret, ret_string = commands.getstatusoutput(command_str)

    if ret == 0:
        return True, ''
    return False, "Cannot connected to Server %s, %s" % (server_address, ret_string)

def get_uuid(user_name, server_address, vm_name):
    # list cache
    command_str = 'isr lshoard -l -s ' + server_address + ' -u ' + user_name
    print command_str
    ret, ret_string = commands.getstatusoutput(command_str)
    if ret != 0:
        return False, "Cannot list up VM hoard"

    # find UUID, which has vm_name
    lines = ret_string.split('\n')
    uuid = None
    for index, line in enumerate(lines):
        if line.find(vm_name) != -1 and len(lines) > (index+1):
            uuid = lines[index+1].lstrip().split(" ")[0]

    return uuid

# remove all cache
def remove_cache(user_name, server_address, vm_name):
    uuid = get_uuid(user_name, server_address, vm_name)
    if uuid != None:
        # erase disk cache
        command_str = 'isr rmhoard ' + uuid + ' -s ' + server_address + ' -u ' + user_name
        ret, ret_string = commands.getstatusoutput(command_str)

    # erase memory
    '''
    mem_dir = '/home/krha/.isr/hoard/img/'
    print 'aaaaaaaaaaaaaaaaaaaaaaaaaa'
    if os.path.exists(mem_dir):
        print 'aaaaaaaaaaaaaaaaaaaaaaaaaa'
        command_str = "rm -rf %s*" %(mem_dir)
        print command_str
        ret, ret_string = commands.getstatusoutput(command_str)
    '''

    mem_dir = '/home/krha/.isr/'
    if os.path.exists(mem_dir):
        command_str = "rm -rf %s" % (mem_dir)
        print command_str
        ret, ret_string = commands.getstatusoutput(command_str)

    return True, ''

# resume VM
def resume_vm(user_name, server_address, vm_name):
    time_start = datetime.now()
    time_end = datetime.now()
    time_transfer_start = datetime.now()
    time_transfer_end = datetime.now()
    time_decomp_mem_start = datetime.now()
    time_decomp_mem_end = datetime.now()
    time_kvm_start = datetime.now()
    time_kvm_end = datetime.now()

    command_str = 'isr resume ' + vm_name + ' -s ' + server_address + ' -u ' + user_name + ' -F -D'
    print command_str
    time_start = datetime.now()
    print "VM Resume Process start : " + str(time_start)
    proc = subprocess.Popen(command_str, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    while True:
        time.sleep(0.1)
        output = proc.stdout.readline()
        if len(output.strip()) != 0 and output.find("[krha]") == -1:
            sys.stdout.write(output)

        # time stamping using log from isr_client
        # Not reliable but fast for simple test
        if output.strip().find("Fetching keyring") == 0:
            time_transfer_start = datetime.now()
        elif output.strip().find("Decrypting and uncompressing") == 0:
            time_transfer_end = datetime.now()
            time_decomp_mem_start = datetime.now()
        elif output.strip().find("Updating hoard cache") == 0:
            time_decomp_mem_end = datetime.now()
        elif output.strip().find("Launching KVM") == 0:
            time_kvm_start = datetime.now()
            break;

    # waiting for TCP socket open
    # predefined for test, it is opened at ISR/vmm/kvm
    # So, no multiple ISR Client at one machine
    telnet_port = 9998
    for i in xrange(200):
        command_str = "netstat -an | grep 127.0.0.1:" + str(telnet_port)
        proc = subprocess.Popen(command_str, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        output = proc.stdout.readline()
        if output.find("LISTEN") != -1:
            break;
        time.sleep(0.1)

    # Getting VM Status information through Telnet
    telnet_connection_waiting(telnet_port)
    time_kvm_end = datetime.now()

    '''
    # Waiting for kvm.vnc
    # find UUID, which has vm_name
    uuid = get_uuid(user_name, server_address, vm_name)
    vnc_path = os.path.join(VNC_PATH, uuid, "cfg", "kvm.vnc")
    for i in xrange(200):
        if os.path.exists(vnc_path):
            time_kvm_end = datetime.now()
            print "VM Resume Process End : " + str(time_kvm_end)
            break;
        time.sleep(0.1)
    '''

    # if we wait for process to end, we cannot return to web client
    # ret = proc.wait()
    time_end = datetime.now()
    print "Return from Resume"
    print "[Total Time] : ", str(time_end-time_start)
    print '[Transfer Time(Memory)] : ', str(time_transfer_end-time_transfer_start)
    print '[Decompression Time] : ', str(time_decomp_mem_end-time_decomp_mem_start)
    print '[KVM Time] : ', str(time_kvm_end-time_kvm_start)

# stop VM
def stop_vm(user_name, server_address, vm_name):
    command_str = 'isr clean ' + vm_name + ' -s ' + server_address + ' -u ' + user_name + ' -f'
    print command_str
    proc = subprocess.Popen(command_str, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    proc.stdin.write('y\n')

    return True

# Exit with error message
def exit_error(error_message):
    print 'Error, ', error_message
    sys.exit(1)

def do_cloud_isr(user_name, vm_name, server_address):
    # compile ISR again, because we have multiple version of ISR such as mock android
    # This is not good approach, but easy for simple test
    # I'll gonna erase this script after submission :(
    recompile_isr(ISR_ORIGIN_SRC_PATH)

    # step1. login
    ret, err = login(user_name, server_address)
    if not ret:
        print err
        return False

    # step2. remove all cache
    ret, err = remove_cache(user_name, server_address, vm_name)
    if not ret:
        return False

    # step3. resume VM, wait until finish (close window)
    start_time = datetime.now()
    resume_vm(user_name, server_address, vm_name)
    end_time = datetime.now()

    return True

def do_mobile_isr(user_name, vm_name, server_address):
    # compile ISR again, because we have multiple version of ISR such as mock android
    # This is not good approach, but easy for simple test
    # I'll gonna erase this script after submission :(
    recompile_isr(ISR_ANDROID_SRC_PATH)

    # Change parcel to indicate it to mobile address, which is specified at server_address
    trick_parcel_address(ISR_ANDROID_PARCEL_PATH, vm_name, server_address)

    # step1. remove all cache
    ret, err = remove_cache(user_name, server_address, vm_name)
    if not ret:
        return False

    # step2. resume VM, wait until finish (close window)
    start_time = datetime.now()
    resume_vm(user_name, server_address, vm_name)
    end_time = datetime.now()
    return True

def trick_parcel_address(parcel_dir, vm_name, server_address):
    parcel_path = os.path.join(parcel_dir, vm_name, 'parcel.cfg')
    print parcel_path
    if not os.path.exists(parcel_path):
        print "Error, check you parcel file location : " + parcel_path
        sys.exit(2)
        return False
    lines = []
    fr = open(parcel_path, 'r')
    for line in fr:
        key = line.split("=")[0].strip()
        if key == "RPATH":
            lines.append("RPATH = http://" + server_address + ":8080\n")
        elif key == "SERVER":
            lines.append("SERVER = " + server_address + "\n")
        else:
            lines.append(line)
    fr.close()
    fw = open(parcel_path, 'w')
    fw.write(''.join(lines))
    fw.close()

def isr_clean_all(server_address, user_name):
    global application_names

    # kill all process that has 'isr'
    # I really hate this way, but it is almost only way that I can clean cache
    command_str = 'ps aux | grep isr'
    ret1, ret_string = commands.getstatusoutput(command_str)
    for line in ret_string.split('\n'):
        if line.find('isr') != -1 and line.find('isr_run.py') == -1 and line.find('vi ') == -1:
            pid = re.search('[A-Za-z]+\s+(\d+).*', line).groups(0)[0]
            command_str = 'kill -9 ' + pid
            print 'kill /isr + \t', command_str
            commands.getoutput(command_str)

    for vm_name in application_names:
        ret = stop_vm(user_name, server_address, vm_name)
        ret = remove_cache(user_name, server_address, vm_name)

def print_usage(program_name):
    print 'usage\t: %s [run|clean] [-u username] [-s server_address] ' % program_name
    print 'example\t: isr_run.py run -u cloudlet -s dagama.isr.cs.cmu.edu'

def main(argv):
    global user_name
    global server_address

    if len(argv) < 3:
        print_usage(os.path.basename(argv[0]))
        sys.exit(2)

    operation = argv[1].lower()
    if not operation in ("clean", "run"):
        print "No supporing operation : ", operation
        print_usage(os.path.basename(argv[0]))
        sys.exit(2)

    try:
        optlist, args = getopt.getopt(argv[2:], 'hu:s:', ["help", "user", "server"])
    except getopt.GetoptError, err:
        print str(err)
        print_usage(os.path.basename(argv[0]))
        sys.exit(2)

    # required input variables
    user_name = None
    server_address = None

    # parse argument
    for o, a in optlist:
        if o in ("-h", "--help"):
            print_usage(os.path.basename(argv[0]))
            sys.exit(0)
        elif o in ("-u", "--user"):
            user_name = a
        elif o in ("-s", "--server"):
            server_address = a
        else:
            assert False, "unhandled option"

    if user_name == None or server_address == None:
        print_usage(os.path.basename(argv[0]))
        sys.exit(2)

    # Always strat from the clean state
    isr_clean_all(server_address, user_name)
    if operation == "clean":
        sys.exit(0)

if __name__ == "__main__":
    main(sys.argv)
    #do_mobile_isr(user_name, "moped", server_address)
    app.run(host='0.0.0.0', port=WEB_SERVER_PORT_NUMBER, processes = 10)

<|endoftext|>


<commit_before>import os
import sys
import subprocess
from random import random
import tempfile
from nalaf.learning.taggers import RelationExtractor
from nalaf import print_warning, print_verbose, print_debug, is_verbose_mode
import numpy as np
import scipy
import sklearn
from sklearn import svm
from sklearn.preprocessing import FunctionTransformer, maxabs_scale
from sklearn.preprocessing import MaxAbsScaler
from sklearn.feature_selection import VarianceThreshold
import time
from sklearn.pipeline import make_pipeline

class SklSVM(RelationExtractor):
    """
    Base class to interact with [scikit-learn SVM](http://scikit-learn.org/stable/modules/svm.html#svm),
    concretly [SVC class](http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC)
    This is actually based on [LIBSVM](https://www.csie.ntu.edu.tw/%7Ecjlin/libsvm/).
    """
    # Likely we want to create a hierarchy for ML "Trainers" or similar

    def __init__(self, model_path=None, classification_threshold=0.0, use_tree_kernel=False, preprocess=True, **svc_parameters):
        assert not use_tree_kernel, NotImplementedError
        assert classification_threshold == 0, NotImplementedError

        self.version = "1.0.0"

        if isinstance(preprocess, list):
            self.preprocess = make_pipeline(*preprocess)
        elif preprocess is True:
            self.preprocess = make_pipeline(
                # # See: http://scikit-learn.org/stable/modules/feature_selection.html#removing-features-with-low-variance
                # (lambda p: VarianceThreshold(threshold=(p * (1 - p))))(0.01),

                # See http://stackoverflow.com/a/41601532/341320
                # logtran = FunctionTransformer(snp.log1p, accept_sparse=True, validate=True)
                # X = logtran.transform(X)
                MaxAbsScaler(copy=False),
            )
        else:
            self.preprocess = make_pipeline(None)

        self.model = svm.SVC(**svc_parameters)

    def train(self, training_corpus):
        X, y = self.__convert_edges_features_to_vector_instances(training_corpus)
        print_debug("Train SVC with #samples {} - #features {} - params: {}".format(X.shape[0], X.shape[1], str(self.model.get_params())))
        start = time.time()
        self.model.fit(X, y)
        end = time.time()
        print_debug("SVC train, running time: ", (end - start))

        return self

    def annotate(self, corpus):
        X, y = self.__convert_edges_features_to_vector_instances(corpus)
        y_pred = self.model.predict(X)
        y_size = len(y)
        print_debug("Mean accuracy: {}".format(sum(real == pred for real, pred in zip(y, y_pred)) / y_size))

        for edge, target_pred in zip(corpus.edges(), y_pred):
            edge.pred_target = target_pred

        return corpus.form_predicted_relations()

    # ----------------------------------------------------------------------------------------------------

    def write_vector_instances(self, corpus, global_feature_set):
        num_features = len(global_feature_set)

        X, y, groups = self.__gen_vector_instances(corpus, num_features=num_features)

        for edge in corpus.edges():
            assert(edge.initial_instance_index is not None)
            edge.features_vector = X.getrow(edge.initial_instance_index)

        return X, y, groups

    def __gen_vector_instances(self, corpus, num_features):

        def fun(X, y, corpus):
            groups = {}
            instance_index = -1
            for docid, document in corpus.documents.items():
                groups[docid] = []  # Note, some documents may not generate instances -- Define them with empty lists

                for edge in document.edges():
                    instance_index += 1
                    edge.initial_instance_index = instance_index
                    groups[docid] = groups.get(docid, []) + [instance_index]

                    for f_index, value in edge.features.items():
                        X[instance_index, f_index] = value

                    y[instance_index] = edge.real_target

            return X, y, groups

        return __class__._create_instances(
            num_features=num_features,
            corpus=corpus,
            preprocess=self.preprocess,
            setting_function=fun
        )

    @staticmethod
    def _create_instances(num_features, corpus, preprocess, setting_function):
        """
        rtype: Tuple[scipy.sparse.csr_matrix, List[int]]
        """
        start = time.time()

        num_instances = sum(1 for _ in corpus.edges())

        # We first construct the X matrix of features with the sparse lil_matrix, which is efficient in reshaping its structure dynamically
        # At the end, we convert this to csr_matrix, which is efficient for algebra operations
        # See https://docs.scipy.org/doc/scipy-0.18.1/reference/generated/scipy.sparse.lil_matrix.html#scipy.sparse.lil_matrix
        # See http://scikit-learn.org/stable/modules/svm.html#svm
        X = scipy.sparse.lil_matrix((num_instances, num_features), dtype=np.float64)
        y = np.zeros(num_instances, order='C')  # -- see: http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC

        X, y, groups = setting_function(X, y, corpus)

        X = X.tocsr()

        print_verbose("SVC before preprocessing, #features: {} && max value: {}".format(X.shape[1], max(sklearn.utils.sparsefuncs.min_max_axis(X, axis=0)[1])))
        if preprocess:
            X = preprocess.fit_transform(X)  # __class__._preprocess(X)
            print_debug("SVC after preprocessing, #features: {} && max value: {}".format(X.shape[1], max(sklearn.utils.sparsefuncs.min_max_axis(X, axis=0)[1])))

        end = time.time()
        print_debug("SVC convert instances, running time: ", (end - start))

        return (X, y, groups)

    def __convert_edges_features_to_vector_instances(self, corpus):
        if __class__._vector_instances_already_computed(corpus):
            return __class__._convert_edges_features_reusing_computed_vector_instances(corpus)
        else:
            # + 1 since the keys are 0-indexed, that is a sole feature indexed by 0 means having 1 feature
            return self.__gen_vector_instances(corpus, max(next(corpus.edges()).features.keys()) + 1)

    @staticmethod
    def _vector_instances_already_computed(corpus):
        return next(corpus.edges()).features_vector is not None

    @staticmethod
    def _convert_edges_features_reusing_computed_vector_instances(corpus):
        num_instances = sum(1 for _ in corpus.edges())
        num_features = next(corpus.edges()).features_vector.shape[1]

        X = scipy.sparse.lil_matrix((num_instances, num_features), dtype=np.float64)
        y = np.zeros(num_instances, order='C')  # -- see: http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC

        for edge_index, edge in enumerate(corpus.edges()):
            X[edge_index, :] = edge.features_vector
            y[edge_index] = edge.real_target

        X = X.tocsr()

        return (X, y)
<commit_msg>do the fit preprocessing only in training!<commit_after>import os
import sys
import subprocess
from random import random
import tempfile
from nalaf.learning.taggers import RelationExtractor
from nalaf import print_warning, print_verbose, print_debug, is_verbose_mode
import numpy as np
import scipy
import sklearn
from sklearn import svm
from sklearn.preprocessing import FunctionTransformer, maxabs_scale
from sklearn.preprocessing import MaxAbsScaler
from sklearn.feature_selection import VarianceThreshold
import time
from sklearn.pipeline import make_pipeline

class SklSVM(RelationExtractor):
    """
    Base class to interact with [scikit-learn SVM](http://scikit-learn.org/stable/modules/svm.html#svm),
    concretly [SVC class](http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC)
    This is actually based on [LIBSVM](https://www.csie.ntu.edu.tw/%7Ecjlin/libsvm/).
    """
    # Likely we want to create a hierarchy for ML "Trainers" or similar

    def __init__(self, model_path=None, classification_threshold=0.0, use_tree_kernel=False, preprocess=True, **svc_parameters):
        assert not use_tree_kernel, NotImplementedError
        assert classification_threshold == 0, NotImplementedError

        self.version = "1.0.0"

        if isinstance(preprocess, list):
            self.preprocess = make_pipeline(*preprocess)
        elif preprocess is True:
            self.preprocess = make_pipeline(
                # # See: http://scikit-learn.org/stable/modules/feature_selection.html#removing-features-with-low-variance
                # (lambda p: VarianceThreshold(threshold=(p * (1 - p))))(0.01),

                # See http://stackoverflow.com/a/41601532/341320
                # logtran = FunctionTransformer(snp.log1p, accept_sparse=True, validate=True)
                # X = logtran.transform(X)
                MaxAbsScaler(copy=False),
            )
        else:
            self.preprocess = make_pipeline(None)

        self.model = svm.SVC(**svc_parameters)

    def train(self, training_corpus):
        X, y = self.__convert_edges_features_to_vector_instances(training_corpus)
        X = self.preprocess.fit_transform(X)
        print_debug("SVC after preprocessing, #features: {} && max value: {}".format(X.shape[1], max(sklearn.utils.sparsefuncs.min_max_axis(X, axis=0)[1])))

        print_debug("Train SVC with #samples {} - #features {} - params: {}".format(X.shape[0], X.shape[1], str(self.model.get_params())))
        start = time.time()
        self.model.fit(X, y)
        end = time.time()
        print_debug("SVC train, running time: ", (end - start))

        return self

    def annotate(self, corpus):
        X, y = self.__convert_edges_features_to_vector_instances(corpus)
        X = self.preprocess.transform(X)
        print_debug("SVC after preprocessing, #features: {} && max value: {}".format(X.shape[1], max(sklearn.utils.sparsefuncs.min_max_axis(X, axis=0)[1])))

        y_pred = self.model.predict(X)
        y_size = len(y)
        print_debug("Mean accuracy: {}".format(sum(real == pred for real, pred in zip(y, y_pred)) / y_size))

        for edge, target_pred in zip(corpus.edges(), y_pred):
            edge.pred_target = target_pred

        return corpus.form_predicted_relations()

    # ----------------------------------------------------------------------------------------------------

    def write_vector_instances(self, corpus, global_feature_set):
        num_features = len(global_feature_set)

        X, y, groups = self.__gen_vector_instances(corpus, num_features=num_features)

        for edge in corpus.edges():
            assert(edge.initial_instance_index is not None)
            edge.features_vector = X.getrow(edge.initial_instance_index)

        return X, y, groups

    def __gen_vector_instances(self, corpus, num_features):

        def fun(X, y, corpus):
            groups = {}
            instance_index = -1
            for docid, document in corpus.documents.items():
                groups[docid] = []  # Note, some documents may not generate instances -- Define them with empty lists

                for edge in document.edges():
                    instance_index += 1
                    edge.initial_instance_index = instance_index
                    groups[docid] = groups.get(docid, []) + [instance_index]

                    for f_index, value in edge.features.items():
                        X[instance_index, f_index] = value

                    y[instance_index] = edge.real_target

            return X, y, groups

        return __class__._create_instances(
            num_features=num_features,
            corpus=corpus,
            setting_function=fun
        )

    @staticmethod
    def _create_instances(num_features, corpus, setting_function):
        """
        rtype: Tuple[scipy.sparse.csr_matrix, List[int]]
        """
        start = time.time()

        num_instances = sum(1 for _ in corpus.edges())

        # We first construct the X matrix of features with the sparse lil_matrix, which is efficient in reshaping its structure dynamically
        # At the end, we convert this to csr_matrix, which is efficient for algebra operations
        # See https://docs.scipy.org/doc/scipy-0.18.1/reference/generated/scipy.sparse.lil_matrix.html#scipy.sparse.lil_matrix
        # See http://scikit-learn.org/stable/modules/svm.html#svm
        X = scipy.sparse.lil_matrix((num_instances, num_features), dtype=np.float64)
        y = np.zeros(num_instances, order='C')  # -- see: http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC

        X, y, groups = setting_function(X, y, corpus)

        X = X.tocsr()

        end = time.time()
        print_debug("SVC convert instances, running time: ", (end - start))

        return (X, y, groups)

    def __convert_edges_features_to_vector_instances(self, corpus):
        if __class__._vector_instances_already_computed(corpus):
            return __class__._convert_edges_features_reusing_computed_vector_instances(corpus)
        else:
            # + 1 since the keys are 0-indexed, that is a sole feature indexed by 0 means having 1 feature
            return self.__gen_vector_instances(corpus, max(next(corpus.edges()).features.keys()) + 1)

    @staticmethod
    def _vector_instances_already_computed(corpus):
        return next(corpus.edges()).features_vector is not None

    @staticmethod
    def _convert_edges_features_reusing_computed_vector_instances(corpus):
        num_instances = sum(1 for _ in corpus.edges())
        num_features = next(corpus.edges()).features_vector.shape[1]

        X = scipy.sparse.lil_matrix((num_instances, num_features), dtype=np.float64)
        y = np.zeros(num_instances, order='C')  # -- see: http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC

        for edge_index, edge in enumerate(corpus.edges()):
            X[edge_index, :] = edge.features_vector
            y[edge_index] = edge.real_target

        X = X.tocsr()

        return (X, y)
<|endoftext|>