<commit_before>import logging
import random
import time

logger = logging.getLogger('mopidy.backends.base')

class BasePlaybackController(object):
    """
    :param backend: backend the controller is a part of
    :type backend: :class:`BaseBackend`
    """

    #: Constant representing the paused state.
    PAUSED = u'paused'

    #: Constant representing the playing state.
    PLAYING = u'playing'

    #: Constant representing the stopped state.
    STOPPED = u'stopped'

    #: :class:`True`
    #:     Tracks are removed from the playlist when they have been played.
    #: :class:`False`
    #:     Tracks are not removed from the playlist.
    consume = False

    #: The currently playing or selected track.
    #:
    #: A two-tuple of (CPID integer, :class:`mopidy.models.Track`) or
    #: :class:`None`.
    current_cp_track = None

    #: :class:`True`
    #:     Tracks are selected at random from the playlist.
    #: :class:`False`
    #:     Tracks are played in the order of the playlist.
    random = False

    #: :class:`True`
    #:     The current playlist is played repeatedly. To repeat a single track,
    #:     select both :attr:`repeat` and :attr:`single`.
    #: :class:`False`
    #:     The current playlist is played once.
    repeat = False

    #: :class:`True`
    #:     Playback is stopped after current song, unless in :attr:`repeat`
    #:     mode.
    #: :class:`False`
    #:     Playback continues after current song.
    single = False

    def __init__(self, backend):
        self.backend = backend
        self._state = self.STOPPED
        self._shuffled = []
        self._first_shuffle = True
        self._play_time_accumulated = 0
        self._play_time_started = None

    def destroy(self):
        """
        Cleanup after component.

        May be overridden by subclasses.
        """
        pass

    def _get_cpid(self, cp_track):
        if cp_track is None:
            return None
        return cp_track[0]

    def _get_track(self, cp_track):
        if cp_track is None:
            return None
        return cp_track[1]

    @property
    def current_cpid(self):
        """
        The CPID (current playlist ID) of the currently playing or selected
        track.

        Read-only. Extracted from :attr:`current_cp_track` for convenience.
        """
        return self._get_cpid(self.current_cp_track)

    @property
    def current_track(self):
        """
        The currently playing or selected :class:`mopidy.models.Track`.

        Read-only. Extracted from :attr:`current_cp_track` for convenience.
        """
        return self._get_track(self.current_cp_track)

    @property
    def current_playlist_position(self):
        """
        The position of the current track in the current playlist.

        Read-only.
        """
        if self.current_cp_track is None:
            return None
        try:
            return self.backend.current_playlist.cp_tracks.index(
                self.current_cp_track)
        except ValueError:
            return None

    @property
    def track_at_eot(self):
        """
        The track that will be played at the end of the current track.

        Read-only. A :class:`mopidy.models.Track` extracted from
        :attr:`cp_track_at_eot` for convenience.
        """
        return self._get_track(self.cp_track_at_eot)

    @property
    def cp_track_at_eot(self):
        """
        The track that will be played at the end of the current track.

        Read-only. A two-tuple of (CPID integer, :class:`mopidy.models.Track`).

        Not necessarily the same track as :attr:`cp_track_at_next`.
        """
        cp_tracks = self.backend.current_playlist.cp_tracks

        if not cp_tracks:
            return None

        if self.random and not self._shuffled:
            if self.repeat or self._first_shuffle:
                logger.debug('Shuffling tracks')
                self._shuffled = cp_tracks
                random.shuffle(self._shuffled)
                self._first_shuffle = False

        if self._shuffled:
            return self._shuffled[0]

        if self.current_cp_track is None:
            return cp_tracks[0]

        if self.repeat and self.single:
            return cp_tracks[
                (self.current_playlist_position) % len(cp_tracks)]

        if self.repeat:
            return cp_tracks[
                (self.current_playlist_position + 1) % len(cp_tracks)]

        try:
            return cp_tracks[self.current_playlist_position + 1]
        except IndexError:
            return None

    @property
    def track_at_next(self):
        """
        The track that will be played if calling :meth:`next()`.

        Read-only. A :class:`mopidy.models.Track` extracted from
        :attr:`cp_track_at_next` for convenience.
        """
        return self._get_track(self.cp_track_at_next)

    @property
    def cp_track_at_next(self):
        """
        The track that will be played if calling :meth:`next()`.

        Read-only. A two-tuple of (CPID integer, :class:`mopidy.models.Track`).

        For normal playback this is the next track in the playlist. If repeat
        is enabled the next track can loop around the playlist. When random is
        enabled this should be a random track, all tracks should be played once
        before the list repeats.
        """
        cp_tracks = self.backend.current_playlist.cp_tracks

        if not cp_tracks:
            return None

        if self.random and not self._shuffled:
            if self.repeat or self._first_shuffle:
                logger.debug('Shuffling tracks')
                self._shuffled = cp_tracks
                random.shuffle(self._shuffled)
                self._first_shuffle = False

        if self._shuffled:
            return self._shuffled[0]

        if self.current_cp_track is None:
            return cp_tracks[0]

        if self.repeat:
            return cp_tracks[
                (self.current_playlist_position + 1) % len(cp_tracks)]

        try:
            return cp_tracks[self.current_playlist_position + 1]
        except IndexError:
            return None

    @property
    def track_at_previous(self):
        """
        The track that will be played if calling :meth:`previous()`.

        Read-only. A :class:`mopidy.models.Track` extracted from
        :attr:`cp_track_at_previous` for convenience.
        """
        return self._get_track(self.cp_track_at_previous)

    @property
    def cp_track_at_previous(self):
        """
        The track that will be played if calling :meth:`previous()`.

        A two-tuple of (CPID integer, :class:`mopidy.models.Track`).

        For normal playback this is the previous track in the playlist. If
        random and/or consume is enabled it should return the current track
        instead.
        """
        if self.repeat or self.consume or self.random:
            return self.current_cp_track

        if self.current_cp_track is None or self.current_playlist_position == 0:
            return None

        return self.backend.current_playlist.cp_tracks[
            self.current_playlist_position - 1]

    @property
    def state(self):
        """
        The playback state. Must be :attr:`PLAYING`, :attr:`PAUSED`, or
        :attr:`STOPPED`.

        Possible states and transitions:

        .. digraph:: state_transitions

            "STOPPED" -> "PLAYING" [ label="play" ]
            "PLAYING" -> "STOPPED" [ label="stop" ]
            "PLAYING" -> "PAUSED" [ label="pause" ]
            "PLAYING" -> "PLAYING" [ label="play" ]
            "PAUSED" -> "PLAYING" [ label="resume" ]
            "PAUSED" -> "STOPPED" [ label="stop" ]
        """
        return self._state

    @state.setter
    def state(self, new_state):
        (old_state, self._state) = (self.state, new_state)
        logger.debug(u'Changing state: %s -> %s', old_state, new_state)
        # FIXME _play_time stuff assumes backend does not have a better way of
        # handeling this stuff :/
        if (old_state in (self.PLAYING, self.STOPPED)
                and new_state == self.PLAYING):
            self._play_time_start()
        elif old_state == self.PLAYING and new_state == self.PAUSED:
            self._play_time_pause()
        elif old_state == self.PAUSED and new_state == self.PLAYING:
            self._play_time_resume()

    @property
    def time_position(self):
        """Time position in milliseconds."""
        if self.state == self.PLAYING:
            time_since_started = (self._current_wall_time -
                self._play_time_started)
            return self._play_time_accumulated + time_since_started
        elif self.state == self.PAUSED:
            return self._play_time_accumulated
        elif self.state == self.STOPPED:
            return 0

    def _play_time_start(self):
        self._play_time_accumulated = 0
        self._play_time_started = self._current_wall_time

    def _play_time_pause(self):
        time_since_started = self._current_wall_time - self._play_time_started
        self._play_time_accumulated += time_since_started

    def _play_time_resume(self):
        self._play_time_started = self._current_wall_time

    @property
    def _current_wall_time(self):
        return int(time.time() * 1000)

    def on_end_of_track(self):
        """
        Tell the playback controller that end of track is reached.

        Typically called by :class:`mopidy.process.CoreProcess` after a message
        from a library thread is received.
        """
        if self.state == self.STOPPED:
            return

        original_cp_track = self.current_cp_track
        if self.cp_track_at_eot:
            self.play(self.cp_track_at_eot)

            if self.random and self.current_cp_track in self._shuffled:
                self._shuffled.remove(self.current_cp_track)
        else:
            self.stop()
            self.current_cp_track = None

        if self.consume:
            self.backend.current_playlist.remove(cpid=original_cp_track[0])

        # Notify frontends of the end_of_track event
        self.backend.core_queue.put({
            'to': 'frontend',
            'command': 'end_of_track',
            'track': original_cp_track[1],
        })

    def on_current_playlist_change(self):
        """
        Tell the playback controller that the current playlist has changed.

        Used by :class:`mopidy.backends.base.BaseCurrentPlaylistController`.
        """
        self._first_shuffle = True
        self._shuffled = []

        if not self.backend.current_playlist.cp_tracks:
            self.stop()
            self.current_cp_track = None
        elif (self.current_cp_track not in
                self.backend.current_playlist.cp_tracks):
            self.current_cp_track = None
            self.stop()

    def next(self):
        """Play the next track."""
        if self.state == self.STOPPED:
            return

        if self.cp_track_at_next:
            self.play(self.cp_track_at_next)
        else:
            self.stop()
            self.current_cp_track = None

        if self.random and self.current_cp_track in self._shuffled:
            self._shuffled.remove(self.current_cp_track)

    def pause(self):
        """Pause playback."""
        if self.state == self.PLAYING and self._pause():
            self.state = self.PAUSED

    def _pause(self):
        """
        To be overridden by subclass. Implement your backend's pause
        functionality here.

        :rtype: :class:`True` if successful, else :class:`False`
        """
        raise NotImplementedError

    def play(self, cp_track=None, on_error_step=1):
        """
        Play the given track, or if the given track is :class:`None`, play the
        currently active track.

        :param cp_track: track to play
        :type cp_track: two-tuple (CPID integer, :class:`mopidy.models.Track`)
            or :class:`None`
        :param on_error_step: direction to step at play error, 1 for next
            track (default), -1 for previous track
        :type on_error_step: int, -1 or 1
        """

        if cp_track is not None:
            assert cp_track in self.backend.current_playlist.cp_tracks
        elif not self.current_cp_track:
            cp_track = self.cp_track_at_next

        if self.state == self.PAUSED and cp_track is None:
            self.resume()
        elif cp_track is not None:
            self.current_cp_track = cp_track
            self.state = self.PLAYING
            if not self._play(cp_track[1]):
                if on_error_step == 1:
                    self.next()
                elif on_error_step == -1:
                    self.previous()

        if self.random and self.current_cp_track in self._shuffled:
            self._shuffled.remove(self.current_cp_track)

        # Notify frontends of the now_playing event
        self.backend.core_queue.put({
            'to': 'frontend',
            'command': 'now_playing',
            'track': cp_track[1],
        })

    def _play(self, track):
        """
        To be overridden by subclass. Implement your backend's play
        functionality here.

        :param track: the track to play
        :type track: :class:`mopidy.models.Track`
        :rtype: :class:`True` if successful, else :class:`False`
        """

        raise NotImplementedError

    def previous(self):
        """Play the previous track."""
        if self.cp_track_at_previous is None:
            return
        if self.state == self.STOPPED:
            return
        self.play(self.cp_track_at_previous, on_error_step=-1)

    def resume(self):
        """If paused, resume playing the current track."""
        if self.state == self.PAUSED and self._resume():
            self.state = self.PLAYING

    def _resume(self):
        """
        To be overridden by subclass. Implement your backend's resume
        functionality here.

        :rtype: :class:`True` if successful, else :class:`False`
        """
        raise NotImplementedError

    def seek(self, time_position):
        """
        Seeks to time position given in milliseconds.

        :param time_position: time position in milliseconds
        :type time_position: int
        :rtype: :class:`True` if successful, else :class:`False`
        """
        if not self.backend.current_playlist.tracks:
            return False

        if self.state == self.STOPPED:
            self.play()
        elif self.state == self.PAUSED:
            self.resume()

        if time_position < 0:
            time_position = 0
        elif time_position > self.current_track.length:
            self.next()
            return True

        self._play_time_started = self._current_wall_time
        self._play_time_accumulated = time_position

        return self._seek(time_position)

    def _seek(self, time_position):
        """
        To be overridden by subclass. Implement your backend's seek
        functionality here.

        :param time_position: time position in milliseconds
        :type time_position: int
        :rtype: :class:`True` if successful, else :class:`False`
        """
        raise NotImplementedError

    def stop(self):
        """Stop playing."""
        if self.state != self.STOPPED and self._stop():
            self.state = self.STOPPED

    def _stop(self):
        """
        To be overridden by subclass. Implement your backend's stop
        functionality here.

        :rtype: :class:`True` if successful, else :class:`False`
        """
        raise NotImplementedError
<commit_msg>Issue end_of_track event before we start playing the next track. Fixes 'now playing' on Last.fm for the second track in the playlist.<commit_after>import logging
import random
import time

logger = logging.getLogger('mopidy.backends.base')

class BasePlaybackController(object):
    """
    :param backend: backend the controller is a part of
    :type backend: :class:`BaseBackend`
    """

    #: Constant representing the paused state.
    PAUSED = u'paused'

    #: Constant representing the playing state.
    PLAYING = u'playing'

    #: Constant representing the stopped state.
    STOPPED = u'stopped'

    #: :class:`True`
    #:     Tracks are removed from the playlist when they have been played.
    #: :class:`False`
    #:     Tracks are not removed from the playlist.
    consume = False

    #: The currently playing or selected track.
    #:
    #: A two-tuple of (CPID integer, :class:`mopidy.models.Track`) or
    #: :class:`None`.
    current_cp_track = None

    #: :class:`True`
    #:     Tracks are selected at random from the playlist.
    #: :class:`False`
    #:     Tracks are played in the order of the playlist.
    random = False

    #: :class:`True`
    #:     The current playlist is played repeatedly. To repeat a single track,
    #:     select both :attr:`repeat` and :attr:`single`.
    #: :class:`False`
    #:     The current playlist is played once.
    repeat = False

    #: :class:`True`
    #:     Playback is stopped after current song, unless in :attr:`repeat`
    #:     mode.
    #: :class:`False`
    #:     Playback continues after current song.
    single = False

    def __init__(self, backend):
        self.backend = backend
        self._state = self.STOPPED
        self._shuffled = []
        self._first_shuffle = True
        self._play_time_accumulated = 0
        self._play_time_started = None

    def destroy(self):
        """
        Cleanup after component.

        May be overridden by subclasses.
        """
        pass

    def _get_cpid(self, cp_track):
        if cp_track is None:
            return None
        return cp_track[0]

    def _get_track(self, cp_track):
        if cp_track is None:
            return None
        return cp_track[1]

    @property
    def current_cpid(self):
        """
        The CPID (current playlist ID) of the currently playing or selected
        track.

        Read-only. Extracted from :attr:`current_cp_track` for convenience.
        """
        return self._get_cpid(self.current_cp_track)

    @property
    def current_track(self):
        """
        The currently playing or selected :class:`mopidy.models.Track`.

        Read-only. Extracted from :attr:`current_cp_track` for convenience.
        """
        return self._get_track(self.current_cp_track)

    @property
    def current_playlist_position(self):
        """
        The position of the current track in the current playlist.

        Read-only.
        """
        if self.current_cp_track is None:
            return None
        try:
            return self.backend.current_playlist.cp_tracks.index(
                self.current_cp_track)
        except ValueError:
            return None

    @property
    def track_at_eot(self):
        """
        The track that will be played at the end of the current track.

        Read-only. A :class:`mopidy.models.Track` extracted from
        :attr:`cp_track_at_eot` for convenience.
        """
        return self._get_track(self.cp_track_at_eot)

    @property
    def cp_track_at_eot(self):
        """
        The track that will be played at the end of the current track.

        Read-only. A two-tuple of (CPID integer, :class:`mopidy.models.Track`).

        Not necessarily the same track as :attr:`cp_track_at_next`.
        """
        cp_tracks = self.backend.current_playlist.cp_tracks

        if not cp_tracks:
            return None

        if self.random and not self._shuffled:
            if self.repeat or self._first_shuffle:
                logger.debug('Shuffling tracks')
                self._shuffled = cp_tracks
                random.shuffle(self._shuffled)
                self._first_shuffle = False

        if self._shuffled:
            return self._shuffled[0]

        if self.current_cp_track is None:
            return cp_tracks[0]

        if self.repeat and self.single:
            return cp_tracks[
                (self.current_playlist_position) % len(cp_tracks)]

        if self.repeat:
            return cp_tracks[
                (self.current_playlist_position + 1) % len(cp_tracks)]

        try:
            return cp_tracks[self.current_playlist_position + 1]
        except IndexError:
            return None

    @property
    def track_at_next(self):
        """
        The track that will be played if calling :meth:`next()`.

        Read-only. A :class:`mopidy.models.Track` extracted from
        :attr:`cp_track_at_next` for convenience.
        """
        return self._get_track(self.cp_track_at_next)

    @property
    def cp_track_at_next(self):
        """
        The track that will be played if calling :meth:`next()`.

        Read-only. A two-tuple of (CPID integer, :class:`mopidy.models.Track`).

        For normal playback this is the next track in the playlist. If repeat
        is enabled the next track can loop around the playlist. When random is
        enabled this should be a random track, all tracks should be played once
        before the list repeats.
        """
        cp_tracks = self.backend.current_playlist.cp_tracks

        if not cp_tracks:
            return None

        if self.random and not self._shuffled:
            if self.repeat or self._first_shuffle:
                logger.debug('Shuffling tracks')
                self._shuffled = cp_tracks
                random.shuffle(self._shuffled)
                self._first_shuffle = False

        if self._shuffled:
            return self._shuffled[0]

        if self.current_cp_track is None:
            return cp_tracks[0]

        if self.repeat:
            return cp_tracks[
                (self.current_playlist_position + 1) % len(cp_tracks)]

        try:
            return cp_tracks[self.current_playlist_position + 1]
        except IndexError:
            return None

    @property
    def track_at_previous(self):
        """
        The track that will be played if calling :meth:`previous()`.

        Read-only. A :class:`mopidy.models.Track` extracted from
        :attr:`cp_track_at_previous` for convenience.
        """
        return self._get_track(self.cp_track_at_previous)

    @property
    def cp_track_at_previous(self):
        """
        The track that will be played if calling :meth:`previous()`.

        A two-tuple of (CPID integer, :class:`mopidy.models.Track`).

        For normal playback this is the previous track in the playlist. If
        random and/or consume is enabled it should return the current track
        instead.
        """
        if self.repeat or self.consume or self.random:
            return self.current_cp_track

        if self.current_cp_track is None or self.current_playlist_position == 0:
            return None

        return self.backend.current_playlist.cp_tracks[
            self.current_playlist_position - 1]

    @property
    def state(self):
        """
        The playback state. Must be :attr:`PLAYING`, :attr:`PAUSED`, or
        :attr:`STOPPED`.

        Possible states and transitions:

        .. digraph:: state_transitions

            "STOPPED" -> "PLAYING" [ label="play" ]
            "PLAYING" -> "STOPPED" [ label="stop" ]
            "PLAYING" -> "PAUSED" [ label="pause" ]
            "PLAYING" -> "PLAYING" [ label="play" ]
            "PAUSED" -> "PLAYING" [ label="resume" ]
            "PAUSED" -> "STOPPED" [ label="stop" ]
        """
        return self._state

    @state.setter
    def state(self, new_state):
        (old_state, self._state) = (self.state, new_state)
        logger.debug(u'Changing state: %s -> %s', old_state, new_state)
        # FIXME _play_time stuff assumes backend does not have a better way of
        # handeling this stuff :/
        if (old_state in (self.PLAYING, self.STOPPED)
                and new_state == self.PLAYING):
            self._play_time_start()
        elif old_state == self.PLAYING and new_state == self.PAUSED:
            self._play_time_pause()
        elif old_state == self.PAUSED and new_state == self.PLAYING:
            self._play_time_resume()

    @property
    def time_position(self):
        """Time position in milliseconds."""
        if self.state == self.PLAYING:
            time_since_started = (self._current_wall_time -
                self._play_time_started)
            return self._play_time_accumulated + time_since_started
        elif self.state == self.PAUSED:
            return self._play_time_accumulated
        elif self.state == self.STOPPED:
            return 0

    def _play_time_start(self):
        self._play_time_accumulated = 0
        self._play_time_started = self._current_wall_time

    def _play_time_pause(self):
        time_since_started = self._current_wall_time - self._play_time_started
        self._play_time_accumulated += time_since_started

    def _play_time_resume(self):
        self._play_time_started = self._current_wall_time

    @property
    def _current_wall_time(self):
        return int(time.time() * 1000)

    def on_end_of_track(self):
        """
        Tell the playback controller that end of track is reached.

        Typically called by :class:`mopidy.process.CoreProcess` after a message
        from a library thread is received.
        """
        if self.state == self.STOPPED:
            return

        original_cp_track = self.current_cp_track

        # Notify frontends of the end_of_track event
        self.backend.core_queue.put({
            'to': 'frontend',
            'command': 'end_of_track',
            'track': original_cp_track[1],
        })

        if self.cp_track_at_eot:
            self.play(self.cp_track_at_eot)

            if self.random and self.current_cp_track in self._shuffled:
                self._shuffled.remove(self.current_cp_track)
        else:
            self.stop()
            self.current_cp_track = None

        if self.consume:
            self.backend.current_playlist.remove(cpid=original_cp_track[0])

    def on_current_playlist_change(self):
        """
        Tell the playback controller that the current playlist has changed.

        Used by :class:`mopidy.backends.base.BaseCurrentPlaylistController`.
        """
        self._first_shuffle = True
        self._shuffled = []

        if not self.backend.current_playlist.cp_tracks:
            self.stop()
            self.current_cp_track = None
        elif (self.current_cp_track not in
                self.backend.current_playlist.cp_tracks):
            self.current_cp_track = None
            self.stop()

    def next(self):
        """Play the next track."""
        if self.state == self.STOPPED:
            return

        if self.cp_track_at_next:
            self.play(self.cp_track_at_next)
        else:
            self.stop()
            self.current_cp_track = None

        if self.random and self.current_cp_track in self._shuffled:
            self._shuffled.remove(self.current_cp_track)

    def pause(self):
        """Pause playback."""
        if self.state == self.PLAYING and self._pause():
            self.state = self.PAUSED

    def _pause(self):
        """
        To be overridden by subclass. Implement your backend's pause
        functionality here.

        :rtype: :class:`True` if successful, else :class:`False`
        """
        raise NotImplementedError

    def play(self, cp_track=None, on_error_step=1):
        """
        Play the given track, or if the given track is :class:`None`, play the
        currently active track.

        :param cp_track: track to play
        :type cp_track: two-tuple (CPID integer, :class:`mopidy.models.Track`)
            or :class:`None`
        :param on_error_step: direction to step at play error, 1 for next
            track (default), -1 for previous track
        :type on_error_step: int, -1 or 1
        """

        if cp_track is not None:
            assert cp_track in self.backend.current_playlist.cp_tracks
        elif not self.current_cp_track:
            cp_track = self.cp_track_at_next

        if self.state == self.PAUSED and cp_track is None:
            self.resume()
        elif cp_track is not None:
            self.current_cp_track = cp_track
            self.state = self.PLAYING
            if not self._play(cp_track[1]):
                if on_error_step == 1:
                    self.next()
                elif on_error_step == -1:
                    self.previous()

        if self.random and self.current_cp_track in self._shuffled:
            self._shuffled.remove(self.current_cp_track)

        # Notify frontends of the now_playing event
        self.backend.core_queue.put({
            'to': 'frontend',
            'command': 'now_playing',
            'track': cp_track[1],
        })

    def _play(self, track):
        """
        To be overridden by subclass. Implement your backend's play
        functionality here.

        :param track: the track to play
        :type track: :class:`mopidy.models.Track`
        :rtype: :class:`True` if successful, else :class:`False`
        """

        raise NotImplementedError

    def previous(self):
        """Play the previous track."""
        if self.cp_track_at_previous is None:
            return
        if self.state == self.STOPPED:
            return
        self.play(self.cp_track_at_previous, on_error_step=-1)

    def resume(self):
        """If paused, resume playing the current track."""
        if self.state == self.PAUSED and self._resume():
            self.state = self.PLAYING

    def _resume(self):
        """
        To be overridden by subclass. Implement your backend's resume
        functionality here.

        :rtype: :class:`True` if successful, else :class:`False`
        """
        raise NotImplementedError

    def seek(self, time_position):
        """
        Seeks to time position given in milliseconds.

        :param time_position: time position in milliseconds
        :type time_position: int
        :rtype: :class:`True` if successful, else :class:`False`
        """
        if not self.backend.current_playlist.tracks:
            return False

        if self.state == self.STOPPED:
            self.play()
        elif self.state == self.PAUSED:
            self.resume()

        if time_position < 0:
            time_position = 0
        elif time_position > self.current_track.length:
            self.next()
            return True

        self._play_time_started = self._current_wall_time
        self._play_time_accumulated = time_position

        return self._seek(time_position)

    def _seek(self, time_position):
        """
        To be overridden by subclass. Implement your backend's seek
        functionality here.

        :param time_position: time position in milliseconds
        :type time_position: int
        :rtype: :class:`True` if successful, else :class:`False`
        """
        raise NotImplementedError

    def stop(self):
        """Stop playing."""
        if self.state != self.STOPPED and self._stop():
            self.state = self.STOPPED

    def _stop(self):
        """
        To be overridden by subclass. Implement your backend's stop
        functionality here.

        :rtype: :class:`True` if successful, else :class:`False`
        """
        raise NotImplementedError
<|endoftext|>


<commit_before>import numpy
import os

import chainer
from chainer import cuda
from chainer.training import extension
from chainer.training import trigger as trigger_module

try:
    import matplotlib  # NOQA
    import matplotlib.pyplot as plt  # NOQA
    matplotlib_color_c0 = matplotlib.colors.to_rgba('#1f77b4')
    # matplotlib_color_c1 = matplotlib.colors.to_rgba('#ff7f0e')
    # chainer_color = matplotlib.colors.to_rgba('#e60312')
    plot_color = matplotlib_color_c0
    plot_color_trans = matplotlib_color_c0[:3] + (0.2,)
    plot_common_kwargs = {
        'alpha': 0.2, 'linewidth': 0, 'color': plot_color_trans}
    _available = True

except (ImportError, TypeError):
    _available = False

def _get_variables(x, memo=None):
    if memo is None:
        memo = ()
    if isinstance(x, chainer.Variable):
        memo += (x,)
    elif isinstance(x, chainer.Link):
        memo += tuple(x.params(include_uninit=True))
    elif isinstance(x, (list, tuple)):
        for xi in x:
            memo += _get_variables(xi)
    return memo

class Reservoir(object):

    """Reservoir sample set with a fixed sized buffer."""

    def __init__(self, size, data_shape, dtype='f'):
        self.size = size
        self.data = numpy.zeros((size, *data_shape), dtype=dtype)
        self.idxs = numpy.zeros((size,), dtype='i')
        self.counter = 0

    def add(self, x, idx=None):
        if self.counter < self.size:
            self.data[self.counter] = x
            self.idxs[self.counter] = idx or self.counter
        elif self.counter >= self.size and \
                numpy.random.random() < self.size / float(self.counter + 1):
            i = numpy.random.randint(self.size)
            self.data[i] = x
            self.idxs[i] = idx or self.counter
        self.counter += 1

    def get_data(self):
        idxs = self.idxs[:min(self.counter, self.size)]
        sorted_args = numpy.argsort(idxs)
        return idxs[sorted_args], self.data[sorted_args]

class Statistician(object):

    """Helper to compute basic NumPy-like statistics."""

    percentiles = (0.13, 2.28, 15.87, 50, 84.13, 97.72, 99.87)

    def __call__(self, x, axis=0, xp=None):
        if not isinstance(axis, (tuple, list)):
            axis = axis,
        shape = (self.data_size,)
        for i, dim in enumerate(x.shape):
            if i not in axis:
                shape += dim,

        xp = xp or cuda.get_array_module(x)
        ret = xp.empty(shape, dtype=xp.float32)
        ret[0] = x.mean(axis=axis)
        ret[1] = x.std(axis=axis)
        ret[2] = x.min(axis=axis)
        ret[3] = x.max(axis=axis)
        if xp == numpy:
            ret[4:] = numpy.percentile(x, self.percentiles, axis=axis)
        else:
            # Copy to host since percentile() is not yet implemented in CuPy
            ret[4:] = cuda.to_gpu(
                numpy.percentile(cuda.to_cpu(x), self.percentiles, axis=axis))

        return xp.rollaxis(ret, 0, ret.ndim)

    @property
    def data_size(self):
        # mean, std, min, max and percentiles
        return 4 + len(self.percentiles)

class VariableStatisticsPlot(extension.Extension):

    """Trainer extension to plot statistics for :class:`Variable`s.

    This extension collects statistics for a single :class:`Variable`, a list
    of :class:`Variables`s or similarly a single or a list of
    :class:`Link`s containing one or more :class:`Variable`s. In case multiple
    :class:`Variable`s are found, the means are computed. The collected
    statistics are plotted and saved as an image in the directory specified by
    the :class:`Trainer`.

    Statistics include mean, standard deviation, minimum, maximum and
    percentiles.

    This extension uses reservoir sampling to preserve memory, using a
    fixed size sample. This means that collected items in the sample are
    discarded uniformly at random when the number of items becomes larger
    than the maximum sample size, but each item is expected to occur in the
    sample with equal probability.

    Args:
        targets (:class:`Variable`, :class:`Link` or list of either):
            Parameters for which statistics are collected.
        max_sample_size (int): Maximum number of samples.
        report_data (bool):  If ``True``, data statistics are plotted.
            If ``False``, they are neither computed or plotted.
        report_grad (bool):  If ``True``, gradient statistics are plotted.
            If ``False``, they are neither computed or plotted.
        trigger: Trigger that decides when to save the plots as an image.
            This is distinct from the trigger of this extension
            itself. If it is a tuple in the form ``<int>, 'epoch'`` or
            ``<int>, 'iteration'``, it is passed to :class:`IntervalTrigger`.
        file_name (str): Name of the plot file under the output directory.
        figsize (tuple of int): Matlotlib ``figsize`` argument that specifies
            the size of the output image.
        marker (str): Matplotlib ``marker`` argument that specified the marker
            style of the plots.
        grid (bool): Matplotlib ``grid`` argument that specifies whether grids
            are rendered in in the plots or not.
    """

    def __init__(self, targets, max_sample_size=1000,
                 report_data=True, report_grad=True,
                 trigger=(1, 'epoch'), file_name='statistics.png',
                 figsize=None, marker=None, grid=True):

        if file_name is None:
            msg = 'Missing output file name of statstics plot'
            raise ValueError(msg)

        self._vars = _get_variables(targets)
        if len(self._vars) == 0:
            msg = 'Need > 0 variables for which to collect statistics.\n' \
                  'Actual: 0'
            raise ValueError(msg)

        self._report_data = report_data
        self._report_grad = report_grad
        self._trigger = trigger_module.get_trigger(trigger)

        self._file_name = file_name
        self._figsize = figsize
        self._marker = marker
        self._grid = grid

        self._keys = []
        if report_data:
            self._keys.append('data')
        if report_grad:
            self._keys.append('grad')

        self._statistician = Statistician()
        self._data_size = self._statistician.data_size
        self._samples = Reservoir(
            max_sample_size, data_shape=(len(self._keys), self._data_size))

    def __call__(self, trainer):
        if not _available:
            # This extension does nothing if matplotlib is not available
            return

        xp = cuda.get_array_module(self._vars[0].data)

        stats = None
        for var in self._vars:
            xs = list(getattr(var, k)[None, :] for k in self._keys)
            xs = xp.concatenate(xs, axis=0)
            stat = self._statistician(
                xs, axis=tuple(range(1, xs.ndim)), xp=xp)
            if stats is None:
                stats = stat
            else:
                stats += stat
        stats = cuda.to_cpu(stats) / len(self._vars)
        self._samples.add(stats, idx=trainer.updater.iteration)

        if self._trigger(trainer):
            file_path = os.path.join(trainer.out, self._file_name)
            self.save_plot(file_path)

    def save_plot(self, file_path):
        nrows = 2  # a mean, std plot and a percentiles plot
        ncols = len(self._keys)
        fig, axes = plt.subplots(
            nrows, ncols, figsize=self._figsize, sharex=True)
        if axes.ndim == 1:
            axes = axes[:, None]

        idxs, data = self._samples.get_data()

        for i in range(ncols):
            ax = axes[0, i]
            ax.errorbar(
                idxs, data[:, i, 0], data[:, i, 1],
                color=plot_color, ecolor=plot_color_trans,
                label='mean, std', marker=self._marker)
            ax.set_title(self._keys[i])

            ax = axes[1, i]
            ax.fill_between(
                idxs, data[:, i, 2], data[:, i, 3], **plot_common_kwargs)
            ax.fill_between(
                idxs, data[:, i, 4], data[:, i, -1], **plot_common_kwargs)
            ax.fill_between(
                idxs, data[:, i, 5], data[:, i, -2], **plot_common_kwargs)
            ax.fill_between(
                idxs, data[:, i, 6], data[:, i, -3], **plot_common_kwargs)
            ax.plot(
                idxs, data[:, i, 7], color=plot_color,
                label='percentiles', marker=self._marker)
            ax.set_xlabel('iteration')

        for ax in axes.ravel():
            ax.legend()
            if self._grid:
                ax.grid()
                ax.set_axisbelow(True)

        fig.savefig(file_path)
        plt.close()
<commit_msg>Fix wrong computation of stats<commit_after>import numpy
import os

import chainer
from chainer import cuda
from chainer.training import extension
from chainer.training import trigger as trigger_module

try:
    import matplotlib  # NOQA
    import matplotlib.pyplot as plt  # NOQA
    matplotlib_color_c0 = matplotlib.colors.to_rgba('#1f77b4')
    # matplotlib_color_c1 = matplotlib.colors.to_rgba('#ff7f0e')
    # chainer_color = matplotlib.colors.to_rgba('#e60312')
    plot_color = matplotlib_color_c0
    plot_color_trans = matplotlib_color_c0[:3] + (0.2,)
    plot_common_kwargs = {
        'alpha': 0.2, 'linewidth': 0, 'color': plot_color_trans}
    _available = True

except (ImportError, TypeError):
    _available = False

def _get_variables(x, memo=None):
    if memo is None:
        memo = ()
    if isinstance(x, chainer.Variable):
        memo += (x,)
    elif isinstance(x, chainer.Link):
        memo += tuple(x.params(include_uninit=True))
    elif isinstance(x, (list, tuple)):
        for xi in x:
            memo += _get_variables(xi)
    return memo

class Reservoir(object):

    """Reservoir sample set with a fixed sized buffer."""

    def __init__(self, size, data_shape, dtype='f'):
        self.size = size
        self.data = numpy.zeros((size, *data_shape), dtype=dtype)
        self.idxs = numpy.zeros((size,), dtype='i')
        self.counter = 0

    def add(self, x, idx=None):
        if self.counter < self.size:
            self.data[self.counter] = x
            self.idxs[self.counter] = idx or self.counter
        elif self.counter >= self.size and \
                numpy.random.random() < self.size / float(self.counter + 1):
            i = numpy.random.randint(self.size)
            self.data[i] = x
            self.idxs[i] = idx or self.counter
        self.counter += 1

    def get_data(self):
        idxs = self.idxs[:min(self.counter, self.size)]
        sorted_args = numpy.argsort(idxs)
        return idxs[sorted_args], self.data[sorted_args]

class Statistician(object):

    """Helper to compute basic NumPy-like statistics."""

    percentiles = (0.13, 2.28, 15.87, 50, 84.13, 97.72, 99.87)

    def __call__(self, x, axis=0, xp=None):
        if not isinstance(axis, (tuple, list)):
            axis = axis,
        shape = (self.data_size,)
        for i, dim in enumerate(x.shape):
            if i not in axis:
                shape += dim,

        xp = xp or cuda.get_array_module(x)
        ret = xp.empty(shape, dtype=xp.float32)
        ret[0] = x.mean(axis=axis)
        ret[1] = x.std(axis=axis)
        ret[2] = x.min(axis=axis)
        ret[3] = x.max(axis=axis)
        if xp == numpy:
            ret[4:] = numpy.percentile(x, self.percentiles, axis=axis)
        else:
            # Copy to host since percentile() is not yet implemented in CuPy
            ret[4:] = cuda.to_gpu(
                numpy.percentile(cuda.to_cpu(x), self.percentiles, axis=axis))

        return xp.rollaxis(ret, 0, ret.ndim)

    @property
    def data_size(self):
        # mean, std, min, max and percentiles
        return 4 + len(self.percentiles)

class VariableStatisticsPlot(extension.Extension):

    """Trainer extension to plot statistics for :class:`Variable`s.

    This extension collects statistics for a single :class:`Variable`, a list
    of :class:`Variables`s or similarly a single or a list of
    :class:`Link`s containing one or more :class:`Variable`s. In case multiple
    :class:`Variable`s are found, the means are computed. The collected
    statistics are plotted and saved as an image in the directory specified by
    the :class:`Trainer`.

    Statistics include mean, standard deviation, minimum, maximum and
    percentiles.

    This extension uses reservoir sampling to preserve memory, using a
    fixed size sample. This means that collected items in the sample are
    discarded uniformly at random when the number of items becomes larger
    than the maximum sample size, but each item is expected to occur in the
    sample with equal probability.

    Args:
        targets (:class:`Variable`, :class:`Link` or list of either):
            Parameters for which statistics are collected.
        max_sample_size (int): Maximum number of samples.
        report_data (bool):  If ``True``, data statistics are plotted.
            If ``False``, they are neither computed or plotted.
        report_grad (bool):  If ``True``, gradient statistics are plotted.
            If ``False``, they are neither computed or plotted.
        trigger: Trigger that decides when to save the plots as an image.
            This is distinct from the trigger of this extension
            itself. If it is a tuple in the form ``<int>, 'epoch'`` or
            ``<int>, 'iteration'``, it is passed to :class:`IntervalTrigger`.
        file_name (str): Name of the plot file under the output directory.
        figsize (tuple of int): Matlotlib ``figsize`` argument that specifies
            the size of the output image.
        marker (str): Matplotlib ``marker`` argument that specified the marker
            style of the plots.
        grid (bool): Matplotlib ``grid`` argument that specifies whether grids
            are rendered in in the plots or not.
    """

    def __init__(self, targets, max_sample_size=1000,
                 report_data=True, report_grad=True,
                 trigger=(1, 'epoch'), file_name='statistics.png',
                 figsize=None, marker=None, grid=True):

        if file_name is None:
            msg = 'Missing output file name of statstics plot'
            raise ValueError(msg)

        self._vars = _get_variables(targets)
        if len(self._vars) == 0:
            msg = 'Need > 0 variables for which to collect statistics.\n' \
                  'Actual: 0'
            raise ValueError(msg)

        self._report_data = report_data
        self._report_grad = report_grad
        self._trigger = trigger_module.get_trigger(trigger)

        self._file_name = file_name
        self._figsize = figsize
        self._marker = marker
        self._grid = grid

        self._keys = []
        if report_data:
            self._keys.append('data')
        if report_grad:
            self._keys.append('grad')

        self._statistician = Statistician()
        self._data_size = self._statistician.data_size
        self._samples = Reservoir(
            max_sample_size, data_shape=(len(self._keys), self._data_size))

    def __call__(self, trainer):
        if not _available:
            # This extension does nothing if matplotlib is not available
            return

        xp = cuda.get_array_module(self._vars[0].data)

        stats = None
        for k in self._keys:
            xs = tuple(getattr(v, k).ravel() for v in self._vars)
            xs = xp.concatenate(xs, axis=0)
            stat = self._statistician(xs, axis=0, xp=xp)
            if stats is None:
                stats = stat[None, :]
            else:
                stats = xp.vstack((stats, stat[None, :]))
        if xp != numpy:
            stats = cuda.to_cpu(stats)
        self._samples.add(stats, idx=trainer.updater.iteration)

        if self._trigger(trainer):
            file_path = os.path.join(trainer.out, self._file_name)
            self.save_plot(file_path)

    def save_plot(self, file_path):
        nrows = 2  # a mean, std plot and a percentiles plot
        ncols = len(self._keys)
        fig, axes = plt.subplots(
            nrows, ncols, figsize=self._figsize, sharex=True)
        if axes.ndim == 1:
            axes = axes[:, None]

        idxs, data = self._samples.get_data()

        for i in range(ncols):
            ax = axes[0, i]
            ax.errorbar(
                idxs, data[:, i, 0], data[:, i, 1],
                color=plot_color, ecolor=plot_color_trans,
                label='mean, std', marker=self._marker)
            ax.set_title(self._keys[i])

            ax = axes[1, i]
            ax.fill_between(
                idxs, data[:, i, 2], data[:, i, 3], **plot_common_kwargs)
            ax.fill_between(
                idxs, data[:, i, 4], data[:, i, -1], **plot_common_kwargs)
            ax.fill_between(
                idxs, data[:, i, 5], data[:, i, -2], **plot_common_kwargs)
            ax.fill_between(
                idxs, data[:, i, 6], data[:, i, -3], **plot_common_kwargs)
            ax.plot(
                idxs, data[:, i, 7], color=plot_color,
                label='percentiles', marker=self._marker)
            ax.set_xlabel('iteration')

        for ax in axes.ravel():
            ax.legend()
            if self._grid:
                ax.grid()
                ax.set_axisbelow(True)

        fig.savefig(file_path)
        plt.close()
<|endoftext|>


<commit_before>"""
Module containing SimConfig class including simulation configuration and methods

"""

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import

# required to make json saving work in Python 2/3
try:
    to_unicode = unicode
except NameError:
    to_unicode = str

from builtins import open
from future import standard_library
standard_library.install_aliases()

from collections import OrderedDict
from .dicts import Dict, ODict

# ----------------------------------------------------------------------------
# SIMULATION CONFIGURATION CLASS
# ----------------------------------------------------------------------------

class SimConfig(object):
    """
    Class for/to <short description of `netpyne.specs.simConfig.SimConfig`>

    """

    def __init__(self, simConfigDict = None):
        # Simulation parameters
        self.duration = self.tstop = 1*1e3 # Duration of the simulation, in ms
        self.dt = 0.025 # Internal integration timestep to use
        self.hParams = Dict({'celsius': 6.3, 'v_init': -65.0, 'clamp_resist': 0.001})  # parameters of h module
        self.cache_efficient = False  # use CVode cache_efficient option to optimize load when running on many cores
        self.cvode_active = False  # Use CVode variable time step
        self.use_fast_imem = False  # use CVode fast_imem to record membrane voltage via i_membrane_
        self.cvode_atol = 0.001  # absolute error tolerance
        self.seeds = Dict({'conn': 1, 'stim': 1, 'loc': 1}) # Seeds for randomizers (connectivity, input stimulation and cell locations)
        self.rand123GlobalIndex = None  # Sets the global index used by all instances of the Random123 instances of Random
        self.createNEURONObj = True  #  create runnable network in NEURON when instantiating netpyne network metadata
        self.createPyStruct = True  # create Python structure (simulator-independent) when instantiating network
        self.addSynMechs = True  # whether to add synaptich mechanisms or not
        self.includeParamsLabel = True  # include label of param rule that created that cell, conn or stim
        self.gatherOnlySimData = False  # omits gathering of net+cell data thus reducing gatherData time
        self.compactConnFormat = False  # replace dict format with compact list format for conns (need to provide list of keys to include)
        self.connRandomSecFromList = True  # select random section (and location) from list even when synsPerConn=1
        self.distributeSynsUniformly = True  # locate synapses at uniformly across section list; if false, place one syn per section in section list
        self.pt3dRelativeToCellLocation = True  # Make cell 3d points relative to the cell x,y,z location
        self.invertedYCoord = True  # Make y-axis coordinate negative so they represent depth when visualized (0 at the top)
        self.allowSelfConns = False  # allow connections from a cell to itself
        self.allowConnsWithWeight0 = True  # allow connections with weight 0
        self.oneSynPerNetcon = True  # create one individual synapse object for each Netcon (if False, same synpase can be shared)
        self.saveCellSecs = True  # save all the sections info for each cell (False reduces time+space; available in netParams; prevents re-simulation)
        self.saveCellConns = True  # save all the conns info for each cell (False reduces time+space; prevents re-simulation)
        self.timing = True  # show timing of each process
        self.saveTiming = False  # save timing data to pickle file
        self.printRunTime = False  # print run time at interval (in sec) specified here (eg. 0.1)
        self.printPopAvgRates = False  # print population avg firing rates after run
        self.printSynsAfterRule = False  # print total of connections after each conn rule is applied
        self.verbose = False  # show detailed messages

        # Recording
        self.recordCells = []  # what cells to record traces from (eg. 'all', 5, or 'PYR')
        self.recordTraces = {}  # Dict of traces to record
        self.recordCellsSpikes = -1  # cells to record spike times from (-1 to record from all)
        self.recordStim = False  # record spikes of cell stims
        self.recordLFP = []  # list of 3D locations to record LFP from
        self.recordDipoles = False # record dipoles
        self.saveLFPCells = False  # Store LFP generate individually by each cell
        self.recordStep = 0.1 # Step size in ms to save data (eg. V traces, LFP, etc)
        self.recordTime = True  # record time step of recording

        # Saving
        self.simLabel = ''  # name of simulation (used as filename if none provided)
        self.saveFolder = ''  # path where to save output data
        self.filename = 'model_output'  # Name of file to save model output (if omitted then saveFolder+simLabel is used)
        self.saveDataInclude = ['netParams', 'netCells', 'netPops', 'simConfig', 'simData']
        self.timestampFilename = False  # Add timestamp to filename to avoid overwriting
        self.savePickle = False # save to pickle file
        self.saveJson = False # save to json file
        self.saveMat = False # save to mat file
        self.saveCSV = False # save to txt file
        self.saveDpk = False # save to .dpk pickled file
        self.saveHDF5 = False # save to HDF5 file
        self.saveDat = False # save traces to .dat file(s)
        self.backupCfgFile = [] # copy cfg file, list with [sourceFile,destFolder] (eg. ['cfg.py', 'backupcfg/'])

        # error checking
        self.checkErrors = False # whether to validate the input parameters (will be turned off if num processors > 1)
        self.checkErrorsVerbose = False # whether to print detailed errors during input parameter validation
        # self.exitOnError = False # whether to hard exit on error

        # Analysis and plotting
        self.analysis = ODict()

        # fill in params from dict passed as argument
        if simConfigDict:
            for k,v in simConfigDict.items():
                if isinstance(v, OrderedDict):
                    setattr(self, k, ODict(v))
                elif isinstance(v, dict):
                    setattr(self, k, Dict(v))
                else:
                    setattr(self, k, v)

    def save(self, filename):
        import os
        basename = os.path.basename(filename)
        folder = filename.split(basename)[0]
        ext = basename.split('.')[1]

        # make dir
        try:
            os.mkdir(folder)
        except OSError:
            if not os.path.exists(folder):
                print(' Could not create', folder)

        dataSave = {'simConfig': self.__dict__}

        # Save to json file
        if ext == 'json':
            from .. import sim
            print(('Saving simConfig to %s ... ' % (filename)))
            sim.saveJSON(filename, dataSave)

    def addAnalysis(self, func, params):
        self.analysis[func] =  params

    def todict(self):
        from ..sim import replaceDictODict
        return replaceDictODict(self.__dict__)
<commit_msg>added cfg method setParam<commit_after>"""
Module containing SimConfig class including simulation configuration and methods

"""

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import

# required to make json saving work in Python 2/3
try:
    to_unicode = unicode
except NameError:
    to_unicode = str

from builtins import open
from future import standard_library
standard_library.install_aliases()

from collections import OrderedDict
from .dicts import Dict, ODict

# ----------------------------------------------------------------------------
# SIMULATION CONFIGURATION CLASS
# ----------------------------------------------------------------------------

class SimConfig(object):
    """
    Class for/to <short description of `netpyne.specs.simConfig.SimConfig`>

    """

    def __init__(self, simConfigDict = None):
        # Simulation parameters
        self.duration = self.tstop = 1*1e3 # Duration of the simulation, in ms
        self.dt = 0.025 # Internal integration timestep to use
        self.hParams = Dict({'celsius': 6.3, 'v_init': -65.0, 'clamp_resist': 0.001})  # parameters of h module
        self.cache_efficient = False  # use CVode cache_efficient option to optimize load when running on many cores
        self.cvode_active = False  # Use CVode variable time step
        self.use_fast_imem = False  # use CVode fast_imem to record membrane voltage via i_membrane_
        self.cvode_atol = 0.001  # absolute error tolerance
        self.seeds = Dict({'conn': 1, 'stim': 1, 'loc': 1}) # Seeds for randomizers (connectivity, input stimulation and cell locations)
        self.rand123GlobalIndex = None  # Sets the global index used by all instances of the Random123 instances of Random
        self.createNEURONObj = True  #  create runnable network in NEURON when instantiating netpyne network metadata
        self.createPyStruct = True  # create Python structure (simulator-independent) when instantiating network
        self.addSynMechs = True  # whether to add synaptich mechanisms or not
        self.includeParamsLabel = True  # include label of param rule that created that cell, conn or stim
        self.gatherOnlySimData = False  # omits gathering of net+cell data thus reducing gatherData time
        self.compactConnFormat = False  # replace dict format with compact list format for conns (need to provide list of keys to include)
        self.connRandomSecFromList = True  # select random section (and location) from list even when synsPerConn=1
        self.distributeSynsUniformly = True  # locate synapses at uniformly across section list; if false, place one syn per section in section list
        self.pt3dRelativeToCellLocation = True  # Make cell 3d points relative to the cell x,y,z location
        self.invertedYCoord = True  # Make y-axis coordinate negative so they represent depth when visualized (0 at the top)
        self.allowSelfConns = False  # allow connections from a cell to itself
        self.allowConnsWithWeight0 = True  # allow connections with weight 0
        self.oneSynPerNetcon = True  # create one individual synapse object for each Netcon (if False, same synpase can be shared)
        self.saveCellSecs = True  # save all the sections info for each cell (False reduces time+space; available in netParams; prevents re-simulation)
        self.saveCellConns = True  # save all the conns info for each cell (False reduces time+space; prevents re-simulation)
        self.timing = True  # show timing of each process
        self.saveTiming = False  # save timing data to pickle file
        self.printRunTime = False  # print run time at interval (in sec) specified here (eg. 0.1)
        self.printPopAvgRates = False  # print population avg firing rates after run
        self.printSynsAfterRule = False  # print total of connections after each conn rule is applied
        self.verbose = False  # show detailed messages

        # Recording
        self.recordCells = []  # what cells to record traces from (eg. 'all', 5, or 'PYR')
        self.recordTraces = {}  # Dict of traces to record
        self.recordCellsSpikes = -1  # cells to record spike times from (-1 to record from all)
        self.recordStim = False  # record spikes of cell stims
        self.recordLFP = []  # list of 3D locations to record LFP from
        self.recordDipoles = False # record dipoles
        self.saveLFPCells = False  # Store LFP generate individually by each cell
        self.recordStep = 0.1 # Step size in ms to save data (eg. V traces, LFP, etc)
        self.recordTime = True  # record time step of recording

        # Saving
        self.simLabel = ''  # name of simulation (used as filename if none provided)
        self.saveFolder = ''  # path where to save output data
        self.filename = 'model_output'  # Name of file to save model output (if omitted then saveFolder+simLabel is used)
        self.saveDataInclude = ['netParams', 'netCells', 'netPops', 'simConfig', 'simData']
        self.timestampFilename = False  # Add timestamp to filename to avoid overwriting
        self.savePickle = False # save to pickle file
        self.saveJson = False # save to json file
        self.saveMat = False # save to mat file
        self.saveCSV = False # save to txt file
        self.saveDpk = False # save to .dpk pickled file
        self.saveHDF5 = False # save to HDF5 file
        self.saveDat = False # save traces to .dat file(s)
        self.backupCfgFile = [] # copy cfg file, list with [sourceFile,destFolder] (eg. ['cfg.py', 'backupcfg/'])

        # error checking
        self.checkErrors = False # whether to validate the input parameters (will be turned off if num processors > 1)
        self.checkErrorsVerbose = False # whether to print detailed errors during input parameter validation
        # self.exitOnError = False # whether to hard exit on error

        # Analysis and plotting
        self.analysis = ODict()

        # fill in params from dict passed as argument
        if simConfigDict:
            for k,v in simConfigDict.items():
                if isinstance(v, OrderedDict):
                    setattr(self, k, ODict(v))
                elif isinstance(v, dict):
                    setattr(self, k, Dict(v))
                else:
                    setattr(self, k, v)

    def save(self, filename):
        import os
        basename = os.path.basename(filename)
        folder = filename.split(basename)[0]
        ext = basename.split('.')[1]

        # make dir
        try:
            os.mkdir(folder)
        except OSError:
            if not os.path.exists(folder):
                print(' Could not create', folder)

        dataSave = {'simConfig': self.__dict__}

        # Save to json file
        if ext == 'json':
            from .. import sim
            print(('Saving simConfig to %s ... ' % (filename)))
            sim.saveJSON(filename, dataSave)

    def setParam(self, param, value):
        self.__dict__[param] = value

    def addAnalysis(self, func, params):
        self.analysis[func] =  params

    def todict(self):
        from ..sim import replaceDictODict
        return replaceDictODict(self.__dict__)
<|endoftext|>


<commit_before>
from __future__ import with_statement

import binascii
import logging
import pykka
import serial
import struct
import threading

import time

logger = logging.getLogger(__name__)

#from mopidy_primare import primare_serial
#pt = primare_serial.PrimareTalker.start(port="/dev/ttyUSB0", input_source=None).proxy()
#pt.power_on()

# Primare documentation on their RS232 protocol writes this:
#  == Command structure ==
#  Commands are sent to the device using the following format, where each field
#  is one byte sent to the device:
#  <STX> <command> <variable> [<value>] <DLE> <ETX>
#  The <command> can be either 'W' for write or 'R' for read. The variable
#  table that follows specifies which variables supports which <command> types.
#  If verbose is active, the device will send replies on the following format
#  either when a command is received or when the variable in question is
#  changed on the device.
#  <STX> <variable> [<value>] <DLE> <ETX>
#  Note that the <value> field can contain several bytes of data for certain
#  commands.
#  == Command special chars ==
#  <STX> = 0x02
#  <DLE> = 0x10
#  <ETX> = 0x03
#  Write = 0x57 (ASCII: W)
#  Read = 0x52 (ASCII: R)
#  == Escape sequence ==
#  If any variable or value byte is equal to <DLE>, this byte must be sent
#  twice to avoid confusing this with end of message.
#  Protocol settings
#  Baud rate: 4800
#  Bits: 8
#  Stop bits: 1
#  Parity: None
#  == Example ==
#  The specific variables and commands will be defined later, here are
#  examples on what the commands looks like in bytes.
#  Command to toggle verbose setting. Command is write, variable is 13 (0x0d)
#  and value is 0.
#  0x02 0x57 0x0xd 0x00 0x10 0x03

# TODO:
# * IMPORTANT: Implement vol_up/vol_down so the default volume can be read by
#       incr and decr volume, instead of setting it to arbitrary default value
#     -- Done, but something is amiss, update to 0.19 first and test again
# * Read out from serial while using remote to see if replies are sent when
#   verbose is enabled
#     -- Replies ARE sent, need another thread to handle input and update status??
#     -- use this for EOL: http://pyserial.sourceforge.net/shortintro.html (pyserial eol)
# * Volume starts at 0? Test using ncmpcpp
# * Better error handling
# * ASAP: Update to 0.19 API
# * ...

class PrimareTalker(pykka.ThreadingActor):
    """
    Independent thread which does the communication with the Primare amplifier.

    Since the communication is done in an independent thread, Mopidy won't
    block other requests while doing rather time consuming work like
    calibrating the Primare amplifier's volume.
    """

    POS_STX = slice(0, 1)
    POS_DLE_ETX = slice(-2, None)
    POS_CMD_VAR = slice(2, 3)
    POS_REPLY_VAR = slice(1, 2)
    POS_REPLY_DATA = slice(2, -2)
    BYTE_STX = '\x02'
    BYTE_WRITE = '\x57'
    BYTE_READ = '\x52'
    BYTE_DLE_ETX = '\x10\x03'

    CMD = 0
    REPLY = 1
    PRIMARE_CMD = {
        'power_toggle': ['0100', '01'],
        'power_on': ['8101', '0101'],
        'power_off': ['8100', '0100'],
        'ir_input_set': ['02YY', '02YY'],
        'input_next': ['0201', '02'],
        'input_prev': ['02FF', '01'],
        'volume_set': ['83FF', '03FF'],
        'volume_up': ['0301', '03'],
        'volume_down': ['03FF', '03'],
        'balance_adjust': ['04YY', '04'],
        'balance_set': ['84YY', '04YY'],
        'mute_toggle': ['0900', '09'],
        'mute_set': ['89YY', '09YY'],
        'dim_cycle': ['0A00', '0A'],
        'dim_set': ['0AYY', '8AYY'],
        'verbose_toggle': ['0D00', '0D'],
        'verbose_set': ['8DYY', '0DYY'],
        'menu_toggle': ['0E01', '0E'],
        'menu_set': ['8EYY', '0EYY'],
        'remote_cmd': ['0FYY', 'YY'],
        'ir_input_toggle': ['1200', '12'],
        'ir_input_set': ['92YY', '12YY'],
        'recall_factory_settings': ['1300', ''],
        'inputname_current_get': ['1400', '14YY'],
        'inputname_specific_get': ['94YY', '94YY'],
        'manufacturer_get': ['1500', '15'],
        'modelname_get': ['16', '16'],
        'swversion_get': ['17', '17']
    }

    # Serial link config
    BAUDRATE = 4800
    BYTESIZE = 8
    PARITY = 'N'
    STOPBITS = 1

    # Timeout in seconds used for read/write operations.
    # If you set the timeout too low, the reads will never get complete
    # confirmations and calibration will decrease volume forever. If you set
    # the timeout too high, stuff takes more time. 0.8s seems like a good value
    # for Primare I22.
    TIMEOUT = 0.8

    # Number of volume levels the amplifier supports.
    # Primare amplifiers have 79 levels
    VOLUME_LEVELS = 79

    def __init__(self, port, input_source):
        super(PrimareTalker, self).__init__()

        self._port = port
        self._input_source = input_source
        self._device = None
        self._lock = threading.Lock()

        # Volume in range 0..VOLUME_LEVELS. :class:`None` before calibration.
        self._volume = None

    def on_start(self):
        self._open_connection()
        #self._print_device_info()
        #self._set_device_to_known_state()
        pass

    # Private methods
    def _open_connection(self):
        logger.info('Primare amplifier: Connecting through "%s"', self._port)
        self._device = serial.Serial(
            port=self._port,
            baudrate=self.BAUDRATE,
            bytesize=self.BYTESIZE,
            parity=self.PARITY,
            stopbits=self.STOPBITS,
            timeout=self.TIMEOUT)

    def _set_device_to_known_state(self):
        self.verbose_set(True)
        self.operate()
        self.input_set(self._input_source)
        self.mute(False)
        self._volume = self._get_current_volume()

    def _print_device_info(self):
        manufacturer = self._send_command('manufacturer_get')
        model = self._send_command('modelname_get')
        swversion = self._send_command('swversion_get')
        inputname = self._send_command('inputname_current_get')

        logger.info("""Connected to:
            Manufacturer:  %s
            Model:         %s
            SW Version:    %s
            Current input: %s """, manufacturer, model, swversion, inputname)

    def _primare_reader(self):
        # The reader will forever do readline, unless _send_command
        # takes the lock to send a command and get a reply
        time.sleep(5)
        print "_primare_reader - starting"
        while(True):
            print "_primare_reader - pre lock"
            with self._lock:
                print "_primare_reader - in lock"
                logger.debug("_primare_reader running")
                reply = self._readline()
                print "_primare_reader - readline"
                if reply != "":
                    _handle_unsolicited_reply()
            print "_primare_reader - post lock"

    def _get_current_volume(self):
        volume = self.volume_down()
        return self.volume_set(volume + 1)

    def _send_command(self, cmd, option=None):
        #if type(value) == unicode:
        #    value = value.encode('utf-8')
        print "_send_command - pre lock"
        print "_send_command - pre lock - lock.locked: %s" % self._lock.locked()
        with self._lock:
            print "_send_command - in lock"
            cmd = PRIMARE_CMD['power_on'][CMD]
            if option is not None:
                cmd = cmd.replace('YY', option)
            self._write(cmd)
            reply = self._readline()
            print "HU HEJ"
        print "_send_command - post lock"

    def _validate_reply(self, cmd, actual_reply_hex, option=None):
        pass

    def _write(self, data):
        # Write data to device.
        if not self._device.isOpen():
            self._device.open()
        # We need to replace single DLE (0x10) with double DLE to discern it
        data = data.replace('10', '1010')
        # Convert ascii string to binary
        data_hex = binascii.unhexlify(data)
        data = BYTE_STX + BYTE_WRITE + data_hex + BYTE_DLE_ETX
        self._device.write(data)
        logger.debug('Write: %s', data)

    def _readline(self):
        # Read line from device.
        if not self._device.isOpen():
            self._device.open()
        eol = binascii.hexlify(BYTE_DLE_ETX)
        result = self._device.readline(eol)
        logger.debug('Read: %s', binascii.hexlify(result)) #if result else "")
        if result:
            reply_string = struct.unpack('c' * len(result), result)
            # reply_string = reply_string.replace('1010', '10') # TODO: How to do this?
            result = ''.join(reply_string[POS_REPLY_DATA])
        return result

    # Public methods
    def power_on(self):
        print "POWER ON"
        self._send_command('power_on')

    def power_off(self):
        self._send_command('power_off')

    def power_toggle(self):
        pass

    def input_set(self, input_source):
        self._send_command('input_set', input_source)

    def input_next(self):
        pass

    def input_prev(self):
        pass

    def volume_get(self):
        return self._volume

    def volume_set(self, volume):
        pass

    def volume_up(self):
        pass

    def volume_down(self):
        pass

    def balance_adjust(self, adjustment):
        pass

    def balance_set(self, balance):
        pass

    def mute_toggle(self):
        pass

    def mute_set(self, mute):
        pass

    def dim_cycle(self):
        pass

    def dim_set(self, level):
        pass

    def verbose_toggle(self):
        pass

    def verbose_set(self, verbose):
        pass

    def menu_toggle(self, verbose):
        pass

    def menu_set(self, menu):
        pass

    def remote_cmd(self, cmd):
        pass

    def ir_input_toggle(self):
        pass

    def ir_input_set(self, ir_input):
        pass

    def recall_factory_settings(self):
        pass

    def manufacturer_get(self):
        pass

    def modelname_get(self):
        pass

    def swversion_get(self):
        pass

    def inputname_current_get(self):
        pass

    def inputname_specific_get(self, input):
        pass<commit_msg>Fix code errors<commit_after>
from __future__ import with_statement

import binascii
import logging
import pykka
import serial
import struct
import threading

import time

logger = logging.getLogger(__name__)

#from mopidy_primare import primare_serial
#pt = primare_serial.PrimareTalker.start(port="/dev/ttyUSB0", input_source=None).proxy()
#pt.power_on()

# Primare documentation on their RS232 protocol writes this:
#  == Command structure ==
#  Commands are sent to the device using the following format, where each field
#  is one byte sent to the device:
#  <STX> <command> <variable> [<value>] <DLE> <ETX>
#  The <command> can be either 'W' for write or 'R' for read. The variable
#  table that follows specifies which variables supports which <command> types.
#  If verbose is active, the device will send replies on the following format
#  either when a command is received or when the variable in question is
#  changed on the device.
#  <STX> <variable> [<value>] <DLE> <ETX>
#  Note that the <value> field can contain several bytes of data for certain
#  commands.
#  == Command special chars ==
#  <STX> = 0x02
#  <DLE> = 0x10
#  <ETX> = 0x03
#  Write = 0x57 (ASCII: W)
#  Read = 0x52 (ASCII: R)
#  == Escape sequence ==
#  If any variable or value byte is equal to <DLE>, this byte must be sent
#  twice to avoid confusing this with end of message.
#  Protocol settings
#  Baud rate: 4800
#  Bits: 8
#  Stop bits: 1
#  Parity: None
#  == Example ==
#  The specific variables and commands will be defined later, here are
#  examples on what the commands looks like in bytes.
#  Command to toggle verbose setting. Command is write, variable is 13 (0x0d)
#  and value is 0.
#  0x02 0x57 0x0xd 0x00 0x10 0x03
POS_STX = slice(0, 1)
POS_DLE_ETX = slice(-2, None)
POS_CMD_VAR = slice(2, 3)
POS_REPLY_VAR = slice(1, 2)
POS_REPLY_DATA = slice(2, -2)
BYTE_STX = '\x02'
BYTE_WRITE = '\x57'
BYTE_READ = '\x52'
BYTE_DLE_ETX = '\x10\x03'

CMD = 0
REPLY = 1
PRIMARE_CMD = {
    'power_toggle': ['0100', '01'],
    'power_on': ['8101', '0101'],
    'power_off': ['8100', '0100'],
    'ir_input_set': ['02YY', '02YY'],
    'input_next': ['0201', '02'],
    'input_prev': ['02FF', '01'],
    'volume_set': ['83FF', '03FF'],
    'volume_up': ['0301', '03'],
    'volume_down': ['03FF', '03'],
    'balance_adjust': ['04YY', '04'],
    'balance_set': ['84YY', '04YY'],
    'mute_toggle': ['0900', '09'],
    'mute_set': ['89YY', '09YY'],
    'dim_cycle': ['0A00', '0A'],
    'dim_set': ['0AYY', '8AYY'],
    'verbose_toggle': ['0D00', '0D'],
    'verbose_set': ['8DYY', '0DYY'],
    'menu_toggle': ['0E01', '0E'],
    'menu_set': ['8EYY', '0EYY'],
    'remote_cmd': ['0FYY', 'YY'],
    'ir_input_toggle': ['1200', '12'],
    'ir_input_set': ['92YY', '12YY'],
    'recall_factory_settings': ['1300', ''],
    'inputname_current_get': ['1400', '14YY'],
    'inputname_specific_get': ['94YY', '94YY'],
    'manufacturer_get': ['1500', '15'],
    'modelname_get': ['16', '16'],
    'swversion_get': ['17', '17']
}
# TODO:
# * IMPORTANT: Implement vol_up/vol_down so the default volume can be read by
#       incr and decr volume, instead of setting it to arbitrary default value
#     -- Done, but something is amiss, update to 0.19 first and test again
# * Read out from serial while using remote to see if replies are sent when
#   verbose is enabled
#     -- Replies ARE sent, need another thread to handle input and update status??
#     -- use this for EOL: http://pyserial.sourceforge.net/shortintro.html (pyserial eol)
# * Volume starts at 0? Test using ncmpcpp
# * Better error handling
# * ASAP: Update to 0.19 API
# * ...

class PrimareTalker(pykka.ThreadingActor):
    """
    Independent thread which does the communication with the Primare amplifier.

    Since the communication is done in an independent thread, Mopidy won't
    block other requests while doing rather time consuming work like
    calibrating the Primare amplifier's volume.
    """

    # Serial link config
    BAUDRATE = 4800
    BYTESIZE = 8
    PARITY = 'N'
    STOPBITS = 1

    # Timeout in seconds used for read/write operations.
    # If you set the timeout too low, the reads will never get complete
    # confirmations and calibration will decrease volume forever. If you set
    # the timeout too high, stuff takes more time. 0.8s seems like a good value
    # for Primare I22.
    TIMEOUT = 0.8

    # Number of volume levels the amplifier supports.
    # Primare amplifiers have 79 levels
    VOLUME_LEVELS = 79

    def __init__(self, port, input_source):
        super(PrimareTalker, self).__init__()

        self._port = port
        self._input_source = input_source
        self._device = None
        self._lock = threading.Lock()

        # Volume in range 0..VOLUME_LEVELS. :class:`None` before calibration.
        self._volume = None

    def on_start(self):
        self._open_connection()
        #self._print_device_info()
        #self._set_device_to_known_state()
        pass

    # Private methods
    def _open_connection(self):
        logger.info('Primare amplifier: Connecting through "%s"', self._port)
        self._device = serial.Serial(
            port=self._port,
            baudrate=self.BAUDRATE,
            bytesize=self.BYTESIZE,
            parity=self.PARITY,
            stopbits=self.STOPBITS,
            timeout=self.TIMEOUT)

    def _set_device_to_known_state(self):
        self.verbose_set(True)
        self.operate()
        self.input_set(self._input_source)
        self.mute(False)
        self._volume = self._get_current_volume()

    def _print_device_info(self):
        manufacturer = self._send_command('manufacturer_get')
        model = self._send_command('modelname_get')
        swversion = self._send_command('swversion_get')
        inputname = self._send_command('inputname_current_get')

        logger.info("""Connected to:
            Manufacturer:  %s
            Model:         %s
            SW Version:    %s
            Current input: %s """, manufacturer, model, swversion, inputname)

    def _primare_reader(self):
        # The reader will forever do readline, unless _send_command
        # takes the lock to send a command and get a reply
        time.sleep(5)
        print "_primare_reader - starting"
        while(True):
            print "_primare_reader - pre lock"
            with self._lock:
                print "_primare_reader - in lock"
                logger.debug("_primare_reader running")
                reply = self._readline()
                print "_primare_reader - readline"
                if reply != "":
                    self._handle_unsolicited_reply()
            print "_primare_reader - post lock"

    def _get_current_volume(self):
        volume = self.volume_down()
        return self.volume_set(volume + 1)

    def _send_command(self, cmd, option=None):
        #if type(value) == unicode:
        #    value = value.encode('utf-8')
        print "_send_command - pre lock"
        print "_send_command - pre lock - lock.locked: %s" % self._lock.locked()
        with self._lock:
            print "_send_command - in lock"
            cmd = PRIMARE_CMD['power_on'][CMD]
            if option is not None:
                cmd = cmd.replace('YY', option)
            self._write(cmd)
            reply = self._readline()
            print "HU HEJ - reply: %s" % binascii.hexlify(reply)
        print "_send_command - post lock"

    def _handle_unsolicited_reply(self):
        pass

    def _validate_reply(self, cmd, actual_reply_hex, option=None):
        pass

    def _write(self, data):
        # Write data to device.
        if not self._device.isOpen():
            self._device.open()
        # We need to replace single DLE (0x10) with double DLE to discern it
        data = data.replace('10', '1010')
        # Convert ascii string to binary
        data_hex = binascii.unhexlify(data)
        data = BYTE_STX + BYTE_WRITE + data_hex + BYTE_DLE_ETX
        self._device.write(data)
        logger.debug('Write: %s', data)

    def _readline(self):
        # Read line from device.
        if not self._device.isOpen():
            self._device.open()
        eol = binascii.hexlify(BYTE_DLE_ETX)
        result = self._device.readline(eol)
        logger.debug('Read: %s', binascii.hexlify(result))  # if result else "")
        if result:
            reply_string = struct.unpack('c' * len(result), result)
            # reply_string = reply_string.replace('1010', '10') # TODO: How to do this?
            result = ''.join(reply_string[POS_REPLY_DATA])
        return result

    # Public methods
    def power_on(self):
        print "POWER ON"
        self._send_command('power_on')

    def power_off(self):
        self._send_command('power_off')

    def power_toggle(self):
        pass

    def input_set(self, input_source):
        self._send_command('input_set', input_source)

    def input_next(self):
        pass

    def input_prev(self):
        pass

    def volume_get(self):
        return self._volume

    def volume_set(self, volume):
        pass

    def volume_up(self):
        pass

    def volume_down(self):
        pass

    def balance_adjust(self, adjustment):
        pass

    def balance_set(self, balance):
        pass

    def mute_toggle(self):
        pass

    def mute_set(self, mute):
        pass

    def dim_cycle(self):
        pass

    def dim_set(self, level):
        pass

    def verbose_toggle(self):
        pass

    def verbose_set(self, verbose):
        pass

    def menu_toggle(self, verbose):
        pass

    def menu_set(self, menu):
        pass

    def remote_cmd(self, cmd):
        pass

    def ir_input_toggle(self):
        pass

    def ir_input_set(self, ir_input):
        pass

    def recall_factory_settings(self):
        pass

    def manufacturer_get(self):
        pass

    def modelname_get(self):
        pass

    def swversion_get(self):
        pass

    def inputname_current_get(self):
        pass

    def inputname_specific_get(self, input):
        pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Imports
from .base_match import BaseMatchAdapter
import nltk.corpus
import nltk.tokenize.punkt
import nltk.stem.snowball
from nltk.corpus import wordnet
import string
from chatterbot.adapters import Adapter

class ApproximateSentenceMatchAdapter(BaseMatchAdapter):
    """
    The Jaccard index is composed of a numerator and denominator.
    In the numerator, we count the number of items that are shared between the sets.
    In the denominator, we count the total number of items across both sets.
    Let’s say we define sentences to be equivalent if 50% or more of their tokens are equivalent.  Here are two sample sentences:
        The young cat is hungry.
        The cat is very hungry.
    When we parse these sentences to remove stopwords, we end up with the following two sets:
        {young, cat, hungry}
        {cat, very, hungry}
    In our example above, our intersection is {cat, hungry}, which has count of two.
    The union of the sets is {young, cat, very, hungry}, which has a count of four.
    Therefore, our Jaccard similarity index is two divided by four, or 50%.
    Given our threshold above, we would consider this to be  a match
    """

    def __init__(self, **kwargs):
        super(ClosestMatchAdapter, self).__init__(**kwargs)
        # Get default English stopwords and extend with punctuation
        self.stopwords = nltk.corpus.stopwords.words('english')
        self.stopwords.extend(string.punctuation)
        self.stopwords.append('')
        self.lemmatizer = nltk.stem.wordnet.WordNetLemmatizer()

    def get_wordnet_pos(self, pos_tag):
        if pos_tag[1].startswith('J'):
            return (pos_tag[0], wordnet.ADJ)
        elif pos_tag[1].startswith('V'):
            return (pos_tag[0], wordnet.VERB)
        elif pos_tag[1].startswith('N'):
            return (pos_tag[0], wordnet.NOUN)
        elif pos_tag[1].startswith('R'):
            return (pos_tag[0], wordnet.ADV)
        else:
            return (pos_tag[0], wordnet.NOUN)

    def is_ci_lemma_stopword_set_match(self,a, b, threshold=0.5):
        """Check if a and b are matches."""
        ratio = 0
        pos_a = map(self.get_wordnet_pos, nltk.pos_tag(nltk.tokenize.word_tokenize(a)))
        pos_b = map(self.get_wordnet_pos, nltk.pos_tag(nltk.tokenize.word_tokenize(b)))
        lemmae_a = [self.lemmatizer.lemmatize(token.lower().strip(string.punctuation), pos) for token, pos in pos_a \
                        if pos == wordnet.NOUN and token.lower().strip(string.punctuation) not in self.stopwords]
        lemmae_b = [self.lemmatizer.lemmatize(token.lower().strip(string.punctuation), pos) for token, pos in pos_b \
                        if pos == wordnet.NOUN and token.lower().strip(string.punctuation) not in self.stopwords]

        # Calculate Jaccard similarity
        try:
            ratio = len(set(lemmae_a).intersection(lemmae_b)) / float(len(set(lemmae_a).union(lemmae_b)))
        except Exception as e:
            print("Error", e)
        return (ratio >= threshold)

    def get(self, input_statement):
        """
        Takes a statement string and a list of statement strings.
        Returns the closest matching statement from the list.
        """
        statement_list = self.context.storage.get_response_statements()

        if not statement_list:
            if self.has_storage_context:
                # Use a randomly picked statement
                return 0, self.context.storage.get_random()
            else:
                raise self.EmptyDatasetException()

        confidence = -1
        sentence_match = input_statement
        # Find the  matching known statement
        for statement in statement_list:
            ratio = self.is_ci_lemma_stopword_set_match(input_statement.text, statement.text)
            if ratio:
                closest_match = statement
            else:
                closest_match = statement
        return 50, closest_match
<commit_msg>Added confidence value 0.5<commit_after># -*- coding: utf-8 -*-
# Imports
from .base_match import BaseMatchAdapter
import nltk.corpus
import nltk.tokenize.punkt
import nltk.stem.snowball
from nltk.corpus import wordnet
import string
from chatterbot.adapters import Adapter

class ApproximateSentenceMatchAdapter(BaseMatchAdapter):
    """
    The Jaccard index is composed of a numerator and denominator.
    In the numerator, we count the number of items that are shared between the sets.
    In the denominator, we count the total number of items across both sets.
    Let’s say we define sentences to be equivalent if 50% or more of their tokens are equivalent.  Here are two sample sentences:
        The young cat is hungry.
        The cat is very hungry.
    When we parse these sentences to remove stopwords, we end up with the following two sets:
        {young, cat, hungry}
        {cat, very, hungry}
    In our example above, our intersection is {cat, hungry}, which has count of two.
    The union of the sets is {young, cat, very, hungry}, which has a count of four.
    Therefore, our Jaccard similarity index is two divided by four, or 50%.
    Given our threshold above, we would consider this to be  a match
    """

    def __init__(self, **kwargs):
        super(ClosestMatchAdapter, self).__init__(**kwargs)
        # Get default English stopwords and extend with punctuation
        self.stopwords = nltk.corpus.stopwords.words('english')
        self.stopwords.extend(string.punctuation)
        self.stopwords.append('')
        self.lemmatizer = nltk.stem.wordnet.WordNetLemmatizer()

    def get_wordnet_pos(self, pos_tag):
        if pos_tag[1].startswith('J'):
            return (pos_tag[0], wordnet.ADJ)
        elif pos_tag[1].startswith('V'):
            return (pos_tag[0], wordnet.VERB)
        elif pos_tag[1].startswith('N'):
            return (pos_tag[0], wordnet.NOUN)
        elif pos_tag[1].startswith('R'):
            return (pos_tag[0], wordnet.ADV)
        else:
            return (pos_tag[0], wordnet.NOUN)

    def is_ci_lemma_stopword_set_match(self,a, b, threshold=0.5):
        """Check if a and b are matches."""
        ratio = 0
        pos_a = map(self.get_wordnet_pos, nltk.pos_tag(nltk.tokenize.word_tokenize(a)))
        pos_b = map(self.get_wordnet_pos, nltk.pos_tag(nltk.tokenize.word_tokenize(b)))
        lemmae_a = [self.lemmatizer.lemmatize(token.lower().strip(string.punctuation), pos) for token, pos in pos_a \
                        if pos == wordnet.NOUN and token.lower().strip(string.punctuation) not in self.stopwords]
        lemmae_b = [self.lemmatizer.lemmatize(token.lower().strip(string.punctuation), pos) for token, pos in pos_b \
                        if pos == wordnet.NOUN and token.lower().strip(string.punctuation) not in self.stopwords]

        # Calculate Jaccard similarity
        try:
            ratio = len(set(lemmae_a).intersection(lemmae_b)) / float(len(set(lemmae_a).union(lemmae_b)))
        except Exception as e:
            print("Error", e)
        return (ratio >= threshold)

    def get(self, input_statement):
        """
        Takes a statement string and a list of statement strings.
        Returns the closest matching statement from the list.
        """
        statement_list = self.context.storage.get_response_statements()

        if not statement_list:
            if self.has_storage_context:
                # Use a randomly picked statement
                return 0, self.context.storage.get_random()
            else:
                raise self.EmptyDatasetException()

        confidence = -1
        sentence_match = input_statement
        # Find the  matching known statement
        for statement in statement_list:
            ratio = self.is_ci_lemma_stopword_set_match(input_statement.text, statement.text)
            if ratio:
                closest_match = statement
            else:
                closest_match = statement
        return 0.5,closest_match
<|endoftext|>