<commit_before>import asyncio

# Constants for pause / resume
PAUSE_ON = 1
PAUSE_OFF = 0

def parse_lines_to_dict(lines):
    """
    Parse a list of message into a dictionnary.
    Used for command like status and stats.

    :param lines: an array of string where each item has the following format
    'name: value'
    :return: a dictionary with the names (as keys) and values found in the
    lines.
    """
    res = {k: v.strip() for k, v in (m.split(':', 1) for m in lines)}
    return res

def parse_to_dicts(lines, containers):
    """
    Parses a list of lines into tuples places in the given containers.

    The lists of lines has the following format

    token1: tval1
    key1: val11
    key2: val12
    token1: tval2
    key1: val21
    key2: val22

    :param lines:
    :param containers: a dictionary { token : array_instance}
    :return:
    """

    pairs = [(a, b.strip()) for a, b in (m.split(':', 1) for m in lines)]
    item = {}
    kind, name = None, None
    for j in range(0, len(pairs)):
        if pairs[j][0] in containers.keys():
            if j != 0:
                containers[kind].append((name, item))
            item = {}
            kind = pairs[j][0]
            name = pairs[j][1]
        else:
            item[pairs[j][0]] = pairs[j][1]
    if kind is not None:
        containers[kind].append((name, item))

    return containers

def parse_playlist(lines):
    files = []
    containers = {'file': files}

    parse_to_dicts(lines, containers)

    return files

def parse_lsinfo(lines):
    dirs = []
    files = []
    playlists = []
    containers = {'directory': dirs,
                  'file': files,
                  'playlist': playlists}

    parse_to_dicts(lines, containers)

    return dirs, files, playlists

def _format_range(start, end):
    """
    Build string for format specification.
    used by many commands, like delete, load, etc...
    :param start:
    :param end:
    :return: the string to be used for range in the command
    """
    if start is None:
        return ''
    if end is None:
        return str(start)+':'
    return str(start)+':'+str(end)

class MpdCommandException(Exception):

    def __init__(self, message, error, line, command, msg):

        super(Exception, self).__init__(message)

        # error num
        self.error = error
        # line of the error in the command list
        self.line = line
        self.command = command
        # Error message.
        self.msg = msg

class MpdClientProtocol(asyncio.StreamReaderProtocol):

    def __init__(self, host=None, port=None, timeout=10, loop=None):

        self.host = host
        self.port = port
        self.timeout = timeout

        self._protocol_version = None
        self.cb_onchange = None

        self.loop = loop
        stream_reader = asyncio.StreamReader(loop=loop)
        super().__init__(stream_reader, self.client_connected, loop)

        self._responses = asyncio.Queue(loop=loop)
        self._cmds = asyncio.Queue(loop=loop)

        self.f_closed = asyncio.Future(loop=loop)
        self.f_stopped = asyncio.Future(loop=loop)

    @property
    def protocol_version(self):
        return self._protocol_version

    @asyncio.coroutine
    def command(self, cmd):
        """
        Send an arbitrary command to mpd.
        This method raises a MpdCommandException if mpd returned a error.

        Example: `client.command(b'play')`

        :param cmd:The command must be a string. If it does not end with
        `\n`, one will be added.
        :return:
        """
        cmd = cmd.encode(encoding='UTF-8')
        if cmd[-1] != b'\n':
            # make sure the command ends with \n, otherwise the client will
            # block
            cmd += b'\n'
        yield from self._cmds.put(cmd)
        resp = yield from self._get_response()
        return resp

    @asyncio.coroutine
    def status(self):
        lines = yield from self.command('status')
        # TODO: used named tuple for status info
        return parse_lines_to_dict(lines)

    @asyncio.coroutine
    def stats(self):
        lines = yield from self.command('stats')
        # TODO: used named tuple for stats info
        return parse_lines_to_dict(lines)

    @asyncio.coroutine
    def clearerror(self):
        """
        Clears the current error message in status.
        This is also accomplished by any command that starts playback.

        :return: True
        """
        yield from self.command('clearerror')
        return True

    @asyncio.coroutine
    def lsinfo(self, path):
        """
        list information.

        :param path:
        :return: a tuple (dirs, files, playlists) corresponding to the
        content of the path.
        dirs, files, playlists are arrays of tuples (path, info) where path
        is the path, relative to the music directory, of the item and info is
        a dictionnary of meta-data for this item.

        """
        resp = yield from self.command('lsinfo "' + path + '"')
        # TODO: improve return format, array of tuples of dict...
        return parse_lsinfo(resp)

    @asyncio.coroutine
    def close(self):
        try:
            yield from self.command('close')
        except MpdCommandException:
            pass
        yield from self.f_stopped

    # Playlist

    def playlistid(self, track_id=None):
        """
        Get information about track(s) in the play queue.

        :param track_id: optional, specifies a single song to display info for.

        :return: an array of tuples `(uri, attrs)` where `uri` is the uri of
        the track and `attrs` is a dictionary with all attributes for the track.
        The only attributes guaranteed to be in `attrs` are `Id` and `Pos`.
        """
        track_id = '' if track_id is None else track_id
        lines = yield from self.command('playlistid {}'.format(track_id))
        return parse_playlist(lines)

    def load(self, playlist, start=None, end=None):
        """
        Loads the playlist into the play queue.
        :param playlist: it can either
        * the name (without the .m3u suffix)of a stored playlist managed by
        mpd (and hence stored inside the configured playlist directory)
        * or the path, relative to the music directory, of a user playlist
        * an uri to a remote playlist

        In these last two cases, supported playlist formats depend on the
        enabled playlist plugins.

        :return:
        """
        track_range = _format_range(start, end)
        yield from self.command('load "{}" {}'.format(playlist, track_range))
        return True

    # Play queue control

    def add(self, uri):
        """
        Adds the track(s) to the playlist.

        :param uri: uri to directory, a single file or a network stream
        File and directory uri are relative to the root of mpd's music
        directory (and must not start with `/`).
        If uri is a directory, all tracks are added recursively. Uri to
        directories must NOT end with '/'.
        When connecting locally (with the socket interface), a 'file://' uri
        can be used to add local files outside the music directory.

        :return: True is the add succeeded, raises an MpdCommandException
        otherwise.
        """
        yield from self.command('add "{}"'.format(uri))
        return True

    def addid(self, uri):
        """
        Adds a track to the playlist (non-recursive) and returns the song id.

        For example:
        `trackid = client.addid( "foo.mp3")`

        :param uri: an uri to a single file or an URL to a network stream.

        :return: the song id in the play queue if the operation succeeded,
        raises an MpdCommandException otherwise.
        """
        resp = yield from self.command('addid "{}"'.format(uri))
        # format : ['Id: 854']
        track_id = resp[0].split(':')[1].strip()
        return track_id

    @asyncio.coroutine
    def clear(self):
        """
        Clears the current play queue.

        :return: True
        """
        yield from self.command('clear')
        return True

    def delete_pos(self, pos):
        """
        Removes track(s) in the playlist.

        If start and end are both given and not None, removes all tracks
        between start and end (not included).
        If only start is given, the track at this position is removed.

        :param pos: index of the track to be removed.
        :return:
        """
        yield from self.command('delete {}'.format(pos))
        return True

    def delete_range(self, start, end=None):
        """
        Removes track(s) in the playlist.

        Removes all tracks between start and end (not included).

        :param start: start of the range of the tracks to be removed.
        :param end: optional,end of the range of the tracks to be removed. If
        end is not given, all tracks from start to the end of the playlist
        will be removed.
        :return:
        """
        track_range = _format_range(start, end)
        yield from self.command('delete {}'.format(track_range))
        return True
    # Controlling playback : pause, next, previous, stop,

    @asyncio.coroutine
    def pause(self, state):
        """
        Pause/resumes playing.
        As the automatic pausing (with no argument) is decrecated in mpd's
        protocole, the state argument is mandatory.
        Returns True or raise an `MpdCommandException`

        :param state: PAUSE_ON or PAUSE_OFF
        :return:
        """
        resp = yield from self.command('pause '+str(state))
        return True

    @asyncio.coroutine
    def next(self):
        """
        Plays next track in the playlist.
        If the end of the playlist is reached, stops playing.
        """
        resp = yield from self.command('next')
        return True

    @asyncio.coroutine
    def previous(self):
        """
        Plays previous track in the playlist.
        If the beginning of the playlist is reached, stay on the first track
        and keeps playing.
        """
        resp = yield from self.command('previous')
        return True

    @asyncio.coroutine
    def play(self, songpos=None):
        """
        Begins playing the playlist at song number `songpos`. If songpos is
        not given, starts playing at current position in the playlist.

        TODO: implement songpos !

        :param songpos: position in the playlist to start playing.
        An `MpdCommandException` will be raised if this index is invalid
        (out of range).
        """
        # TODO: implement songpos !
        if songpos is None:
            resp = yield from self.command('play')
        return True

    @asyncio.coroutine
    def stop(self):
        """
        Stops playback.
        """
        resp = yield from self.command('stop')
        return True

    @asyncio.coroutine
    def _run(self):

        # first, wait for welcome message
        yield from self._welcome_msg()

        while not self.f_closed.done():

            yield from self._send_cmd(b'idle\n')

            f_resp = asyncio.async(self._read_response())
            f_cmd = asyncio.async(self._cmds.get())

            done, pending = yield from \
                asyncio.wait([f_resp, f_cmd, self.f_closed],
                             return_when=asyncio.FIRST_COMPLETED)

            if self.f_closed in done:
                # The socked has be been closed, cancel pending tasks
                f_cmd.cancel()
                f_resp.cancel()
                break

            if f_resp in done:
                # got a notification from our idle wait
                if self.cb_onchange is not None:
                    msg = f_resp.result()
                    self.cb_onchange(msg)

            if f_cmd in done:

                yield from self._send_cmd(b'noidle\n')
                yield from f_resp

                cmd = f_cmd.result()
                yield from self._send_cmd(cmd)
                response = yield from self._read_response()
                yield from self._responses.put(response)
            else:
                f_cmd.cancel()
        self.f_stopped.set_result(True)

    @asyncio.coroutine
    def _welcome_msg(self):
        while True:
            line = yield from self.reader.readline()
            print(line)
            if line.startswith(b'OK MPD'):
                self._protocol_version = line[7:-1]
                break

    @asyncio.coroutine
    def _send_cmd(self, cmd):
        self.writer.write(cmd)
        yield from self.writer.drain()

    @asyncio.coroutine
    def _read_response(self):

        message = []
        while not self.f_closed.done():
            line = yield from self.reader.readline()
            if line == b'':
                if self.reader.at_eof():
                    # When at eof, readline returns empty lines indefinitely.
                    break
            elif line.startswith(b'OK'):
                break
            elif line.startswith(b'ACK'):
                message.append(line[:-1].decode(encoding='UTF-8'))
                break
            else:
                message.append(line[:-1].decode(encoding='UTF-8'))

        return message

    @asyncio.coroutine
    def _get_response(self):
        """
        Get the response from the response queue and raise an Exception if
        mpd returned an error or failure.
        :return:
        """
        resp = yield from self._responses.get()
        if len(resp) == 1:
            line = resp[0]
            if line.startswith('ACK'):
                i = line.index('@')
                error = line[5:i]
                l = line[i+1:line.index(']')]
                command = line[line.index('{'):line.index('}')]
                msg = line[line.index('}')+1:]
                raise MpdCommandException(line, error, l, command, msg)
        return resp

    def client_connected(self, reader, writer):
        # Callback from StreamReaderProtocol
        self.reader = reader
        self.writer = writer
        # Start the task that handles incoming messages.
        self.worker = asyncio.async(self._run(), loop=self.loop)

        # FIXME : be careful, any exception in self.worker would be invisible
        # without this callback !
        self.worker.add_done_callback(self.on_worker)

    def on_worker(self, future):
        print('on worker {}'.format(future))

    def connection_lost(self, exc):
        # When the connection is closed, signal closing,will still need to
        # cancel the prnding task in _run for a clean stop.
        self.f_closed.set_result(True)
        pass

@asyncio.coroutine
def connect(host, port, loop=None):
    if loop is None:
        loop = asyncio.get_event_loop()

    transport, protocol = yield from loop.create_connection(
        lambda: MpdClientProtocol(), host, port)

    return protocol
<commit_msg>Fix parsing of error responses<commit_after>import asyncio

# Constants for pause / resume
PAUSE_ON = 1
PAUSE_OFF = 0

def parse_lines_to_dict(lines):
    """
    Parse a list of message into a dictionnary.
    Used for command like status and stats.

    :param lines: an array of string where each item has the following format
    'name: value'
    :return: a dictionary with the names (as keys) and values found in the
    lines.
    """
    res = {k: v.strip() for k, v in (m.split(':', 1) for m in lines)}
    return res

def parse_to_dicts(lines, containers):
    """
    Parses a list of lines into tuples places in the given containers.

    The lists of lines has the following format

    token1: tval1
    key1: val11
    key2: val12
    token1: tval2
    key1: val21
    key2: val22

    :param lines:
    :param containers: a dictionary { token : array_instance}
    :return:
    """

    pairs = [(a, b.strip()) for a, b in (m.split(':', 1) for m in lines)]
    item = {}
    kind, name = None, None
    for j in range(0, len(pairs)):
        if pairs[j][0] in containers.keys():
            if j != 0:
                containers[kind].append((name, item))
            item = {}
            kind = pairs[j][0]
            name = pairs[j][1]
        else:
            item[pairs[j][0]] = pairs[j][1]
    if kind is not None:
        containers[kind].append((name, item))

    return containers

def parse_playlist(lines):
    files = []
    containers = {'file': files}

    parse_to_dicts(lines, containers)

    return files

def parse_lsinfo(lines):
    dirs = []
    files = []
    playlists = []
    containers = {'directory': dirs,
                  'file': files,
                  'playlist': playlists}

    parse_to_dicts(lines, containers)

    return dirs, files, playlists

def _format_range(start, end):
    """
    Build string for format specification.
    used by many commands, like delete, load, etc...
    :param start:
    :param end:
    :return: the string to be used for range in the command
    """
    if start is None:
        return ''
    if end is None:
        return str(start)+':'
    return str(start)+':'+str(end)

class MpdCommandException(Exception):

    def __init__(self, message, error, line, command, msg):

        super(Exception, self).__init__(message)

        # error num
        self.error = error
        # line of the error in the command list
        self.line = line
        self.command = command
        # Error message.
        self.msg = msg

class MpdClientProtocol(asyncio.StreamReaderProtocol):

    def __init__(self, host=None, port=None, timeout=10, loop=None):

        self.host = host
        self.port = port
        self.timeout = timeout

        self._protocol_version = None
        self.cb_onchange = None

        self.loop = loop
        stream_reader = asyncio.StreamReader(loop=loop)
        super().__init__(stream_reader, self.client_connected, loop)

        self._responses = asyncio.Queue(loop=loop)
        self._cmds = asyncio.Queue(loop=loop)

        self.f_closed = asyncio.Future(loop=loop)
        self.f_stopped = asyncio.Future(loop=loop)

    @property
    def protocol_version(self):
        return self._protocol_version

    @asyncio.coroutine
    def command(self, cmd):
        """
        Send an arbitrary command to mpd.
        This method raises a MpdCommandException if mpd returned a error.

        Example: `client.command(b'play')`

        :param cmd:The command must be a string. If it does not end with
        `\n`, one will be added.
        :return:
        """
        cmd = cmd.encode(encoding='UTF-8')
        if cmd[-1] != b'\n':
            # make sure the command ends with \n, otherwise the client will
            # block
            cmd += b'\n'
        yield from self._cmds.put(cmd)
        resp = yield from self._get_response()
        return resp

    @asyncio.coroutine
    def status(self):
        lines = yield from self.command('status')
        # TODO: used named tuple for status info
        return parse_lines_to_dict(lines)

    @asyncio.coroutine
    def stats(self):
        lines = yield from self.command('stats')
        # TODO: used named tuple for stats info
        return parse_lines_to_dict(lines)

    @asyncio.coroutine
    def clearerror(self):
        """
        Clears the current error message in status.
        This is also accomplished by any command that starts playback.

        :return: True
        """
        yield from self.command('clearerror')
        return True

    @asyncio.coroutine
    def lsinfo(self, path):
        """
        list information.

        :param path:
        :return: a tuple (dirs, files, playlists) corresponding to the
        content of the path.
        dirs, files, playlists are arrays of tuples (path, info) where path
        is the path, relative to the music directory, of the item and info is
        a dictionnary of meta-data for this item.

        """
        resp = yield from self.command('lsinfo "' + path + '"')
        # TODO: improve return format, array of tuples of dict...
        return parse_lsinfo(resp)

    @asyncio.coroutine
    def close(self):
        try:
            yield from self.command('close')
        except MpdCommandException:
            pass
        yield from self.f_stopped

    # Playlist

    def playlistid(self, track_id=None):
        """
        Get information about track(s) in the play queue.

        :param track_id: optional, specifies a single song to display info for.

        :return: an array of tuples `(uri, attrs)` where `uri` is the uri of
        the track and `attrs` is a dictionary with all attributes for the track.
        The only attributes guaranteed to be in `attrs` are `Id` and `Pos`.
        """
        track_id = '' if track_id is None else track_id
        lines = yield from self.command('playlistid {}'.format(track_id))
        return parse_playlist(lines)

    def load(self, playlist, start=None, end=None):
        """
        Loads the playlist into the play queue.
        :param playlist: it can either
        * the name (without the .m3u suffix)of a stored playlist managed by
        mpd (and hence stored inside the configured playlist directory)
        * or the path, relative to the music directory, of a user playlist
        * an uri to a remote playlist

        In these last two cases, supported playlist formats depend on the
        enabled playlist plugins.

        :return:
        """
        track_range = _format_range(start, end)
        yield from self.command('load "{}" {}'.format(playlist, track_range))
        return True

    # Play queue control

    def add(self, uri):
        """
        Adds the track(s) to the playlist.

        :param uri: uri to directory, a single file or a network stream
        File and directory uri are relative to the root of mpd's music
        directory (and must not start with `/`).
        If uri is a directory, all tracks are added recursively. Uri to
        directories must NOT end with '/'.
        When connecting locally (with the socket interface), a 'file://' uri
        can be used to add local files outside the music directory.

        :return: True is the add succeeded, raises an MpdCommandException
        otherwise.
        """
        yield from self.command('add "{}"'.format(uri))
        return True

    def addid(self, uri):
        """
        Adds a track to the playlist (non-recursive) and returns the song id.

        For example:
        `trackid = client.addid( "foo.mp3")`

        :param uri: an uri to a single file or an URL to a network stream.

        :return: the song id in the play queue if the operation succeeded,
        raises an MpdCommandException otherwise.
        """
        resp = yield from self.command('addid "{}"'.format(uri))
        # format : ['Id: 854']
        track_id = resp[0].split(':')[1].strip()
        return track_id

    @asyncio.coroutine
    def clear(self):
        """
        Clears the current play queue.

        :return: True
        """
        yield from self.command('clear')
        return True

    def delete_pos(self, pos):
        """
        Removes track(s) in the playlist.

        If start and end are both given and not None, removes all tracks
        between start and end (not included).
        If only start is given, the track at this position is removed.

        :param pos: index of the track to be removed.
        :return:
        """
        yield from self.command('delete {}'.format(pos))
        return True

    def delete_range(self, start, end=None):
        """
        Removes track(s) in the playlist.

        Removes all tracks between start and end (not included).

        :param start: start of the range of the tracks to be removed.
        :param end: optional,end of the range of the tracks to be removed. If
        end is not given, all tracks from start to the end of the playlist
        will be removed.
        :return:
        """
        track_range = _format_range(start, end)
        yield from self.command('delete {}'.format(track_range))
        return True
    # Controlling playback : pause, next, previous, stop,

    @asyncio.coroutine
    def pause(self, state):
        """
        Pause/resumes playing.
        As the automatic pausing (with no argument) is decrecated in mpd's
        protocole, the state argument is mandatory.
        Returns True or raise an `MpdCommandException`

        :param state: PAUSE_ON or PAUSE_OFF
        :return:
        """
        resp = yield from self.command('pause '+str(state))
        return True

    @asyncio.coroutine
    def next(self):
        """
        Plays next track in the playlist.
        If the end of the playlist is reached, stops playing.
        """
        resp = yield from self.command('next')
        return True

    @asyncio.coroutine
    def previous(self):
        """
        Plays previous track in the playlist.
        If the beginning of the playlist is reached, stay on the first track
        and keeps playing.
        """
        resp = yield from self.command('previous')
        return True

    @asyncio.coroutine
    def play(self, songpos=None):
        """
        Begins playing the playlist at song number `songpos`. If songpos is
        not given, starts playing at current position in the playlist.

        TODO: implement songpos !

        :param songpos: position in the playlist to start playing.
        An `MpdCommandException` will be raised if this index is invalid
        (out of range).
        """
        # TODO: implement songpos !
        if songpos is None:
            resp = yield from self.command('play')
        return True

    @asyncio.coroutine
    def stop(self):
        """
        Stops playback.
        """
        resp = yield from self.command('stop')
        return True

    @asyncio.coroutine
    def _run(self):

        # first, wait for welcome message
        yield from self._welcome_msg()

        while not self.f_closed.done():

            yield from self._send_cmd(b'idle\n')

            f_resp = asyncio.async(self._read_response())
            f_cmd = asyncio.async(self._cmds.get())

            done, pending = yield from \
                asyncio.wait([f_resp, f_cmd, self.f_closed],
                             return_when=asyncio.FIRST_COMPLETED)

            if self.f_closed in done:
                # The socked has be been closed, cancel pending tasks
                f_cmd.cancel()
                f_resp.cancel()
                break

            if f_resp in done:
                # got a notification from our idle wait
                if self.cb_onchange is not None:
                    msg = f_resp.result()
                    self.cb_onchange(msg)

            if f_cmd in done:

                yield from self._send_cmd(b'noidle\n')
                yield from f_resp

                cmd = f_cmd.result()
                yield from self._send_cmd(cmd)
                response = yield from self._read_response()
                yield from self._responses.put(response)
            else:
                f_cmd.cancel()
        self.f_stopped.set_result(True)

    @asyncio.coroutine
    def _welcome_msg(self):
        while True:
            line = yield from self.reader.readline()
            print(line)
            if line.startswith(b'OK MPD'):
                self._protocol_version = line[7:-1]
                break

    @asyncio.coroutine
    def _send_cmd(self, cmd):
        self.writer.write(cmd)
        yield from self.writer.drain()

    @asyncio.coroutine
    def _read_response(self):

        message = []
        while not self.f_closed.done():
            line = yield from self.reader.readline()
            if line == b'':
                if self.reader.at_eof():
                    # When at eof, readline returns empty lines indefinitely.
                    break
            elif line.startswith(b'OK'):
                break
            elif line.startswith(b'ACK'):
                message.append(line[:-1].decode(encoding='UTF-8'))
                break
            else:
                message.append(line[:-1].decode(encoding='UTF-8'))

        return message

    @asyncio.coroutine
    def _get_response(self):
        """
        Get the response from the response queue and raise an Exception if
        mpd returned an error or failure.
        :return:
        """
        resp = yield from self._responses.get()
        if len(resp) == 1:
            line = resp[0]
            if line.startswith('ACK'):
                i = line.index('@')
                error = line[5:i]
                l = line[i+1:line.index(']')]
                command = line[line.index('{')+1:line.index('}')]
                msg = line[line.index('}')+1:]
                raise MpdCommandException(line, error, l, command, msg)
        return resp

    def client_connected(self, reader, writer):
        # Callback from StreamReaderProtocol
        self.reader = reader
        self.writer = writer
        # Start the task that handles incoming messages.
        self.worker = asyncio.async(self._run(), loop=self.loop)

        # FIXME : be careful, any exception in self.worker would be invisible
        # without this callback !
        self.worker.add_done_callback(self.on_worker)

    def on_worker(self, future):
        print('on worker {}'.format(future))

    def connection_lost(self, exc):
        # When the connection is closed, signal closing,will still need to
        # cancel the prnding task in _run for a clean stop.
        self.f_closed.set_result(True)
        pass

@asyncio.coroutine
def connect(host, port, loop=None):
    if loop is None:
        loop = asyncio.get_event_loop()

    transport, protocol = yield from loop.create_connection(
        lambda: MpdClientProtocol(), host, port)

    return protocol
<|endoftext|>


<commit_before>
from EnsoMetricsGraph import EnsoMetricsTable

#EnsoMetrics =[{'col1':'IPSL-CM5A-LR','col2':0.82,'col3':4.1},
#              {'col1':'IPSL-CM5A-MR','col2':1.2,'col3':4.5}]
EnsoMetrics =[[1,2,3],[4,5,6]]

fig=EnsoMetricsTable(EnsoMetrics, 'EnsoMetrics')
<commit_msg>Create metrics table in EnsoMetricsGraph.py<commit_after>
from EnsoMetricsGraph import EnsoMetricsTable

EnsoMetrics =[['IPSL-CM5A-LR','0.82','4.1'],
              ['IPSL-CM5A-MR','1.2','4.5']]
#EnsoMetrics =[[1,2,3],[4,5,6]]

fig=EnsoMetricsTable(EnsoMetrics, 'EnsoMetrics')
<|endoftext|>


<commit_before>import plugins, packetFactory
import data.clients, data.players

userPrefs = {}

@plugins.onStartHook
def createPrefs():
	pass

@plugins.onConnectionHook
def checkConfig(user):
	if user.playerId in data.clients.connectedClients:
		clientPrefs = data.clients.connectedClients[user.playerId].getPrefs()
		if 'globalChat' not in clientPrefs:
			clientPrefs['globalChat'] = True
			user.sendCryptoPacket(packetFactory.ChatPacket(user.playerId, "[Proxy] Global chat is enabled, use |gon to join, use |g <Message> to chat and |goff to disable it.").build())
		data.clients.connectedClients[user.playerId].setPrefs(clientPrefs)

@plugins.commandHook("gon")
def enable(context, params):
	prefs = data.clients.connectedClients[context.playerId].getPrefs()
	prefs['globalChat'] = True
	context.sendCryptoPacket(packetFactory.ChatPacket(context.playerId, "[GlobalChat] Global chat enabled for this session.").build())
	data.clients.connectedClients[context.playerId].setPrefs(prefs)

@plugins.commandHook("goff")
def disable(context, params):
	prefs = data.clients.connectedClients[context.playerId].getPrefs()
	prefs['globalChat'] = False
	context.sendCryptoPacket(packetFactory.ChatPacket(context.playerId, "[GlobalChat] Global chat disabled for this session.").build())
	data.clients.connectedClients[context.playerId].setPrefs(prefs)

@plugins.commandHook("g")
def chat(context, params):
	if not data.clients.connectedClients[context.playerId].getPrefs()['globalChat']:
		context.sendCryptoPacket(packetFactory.ChatPacket(context.playerId, "[GlobalChat] You do not have global chat enabled, and can not send a global message.").build())
		return
	print("[GlobalChat] <%s> %s" % (data.players.playerList[context.playerId][0], params[3:]))
	for client in data.clients.connectedClients.values():
		if client.getPrefs()['globalChat'] and client.getHandle() is not None:
			client.getHandle().sendCryptoPacket(packetFactory.TeamChatPacket(context.playerId, data.players.playerList[context.playerId][0], "[G] %s" % params[3:]).build())
<commit_msg>Make global chat presistant<commit_after>import plugins, packetFactory, json, os
import data.clients, data.players

gcUserPrefs = {}

@plugins.onStartHook
def createPrefs():
	global gcUserPrefs
	if not os.path.exists("cfg/gchat.prefs.json"):
		f = open('cfg/gchat.prefs.json', 'w')
		f.write(json.dumps(gcUserPrefs))
		f.close()
		print("[GlobalChat] Created user prefrences.")
	else:
		f = open('cfg/gchat.prefs.json', 'r')
		prefs = f.read()
		f.close()
		gcUserPrefs = json.loads(prefs)
		print('[GlobalChat] Loaded %i users prefrences.' % len (gcUserPrefs))

@plugins.onConnectionHook
def checkConfig(user):
	global gcUserPrefs
	if user.playerId in data.clients.connectedClients:
		clientPrefs = data.clients.connectedClients[user.playerId].getPrefs()
		if 'globalChat' not in clientPrefs:
			if user.playerId in gcUserPrefs:
				clientPrefs['globalChat'] = gcUserPrefs[user.playerId]['toggle']
			else:
				clientPrefs['globalChat'] = True
				gcUserPrefs[user.playerId]['toggle'] = True
				savePrefs()
			if clientPrefs['globalChat'] == True:
				user.sendCryptoPacket(packetFactory.ChatPacket(user.playerId, "[Proxy] {red}Global chat is enabled, use |g <Message> to chat and |goff to disable it.").build())
			else:
				user.sendCryptoPacket(packetFactory.ChatPacket(user.playerId, "[Proxy] {red}Global chat is disabled, use |gon to enable it and use |g <Message> to chat.").build())
		data.clients.connectedClients[user.playerId].setPrefs(clientPrefs)

def savePrefs():
	f = open('cfg/gchat.prefs.json', 'w')
	f.write(json.dumps(gcUserPrefs))
	f.close()
	print("[GlobalChat] Saved user prefrences.")

@plugins.commandHook("gon")
def enable(context, params):
	global gcUserPrefs
	prefs = data.clients.connectedClients[context.playerId].getPrefs()
	prefs['globalChat'] = True
	context.sendCryptoPacket(packetFactory.ChatPacket(context.playerId, "[GlobalChat] Global chat enabled for you.").build())
	data.clients.connectedClients[context.playerId].setPrefs(prefs)
	gcUserPrefs[context.playerId]['toggle'] = True
	savePrefs()

@plugins.commandHook("goff")
def disable(context, params):
	prefs = data.clients.connectedClients[context.playerId].getPrefs()
	prefs['globalChat'] = False
	context.sendCryptoPacket(packetFactory.ChatPacket(context.playerId, "[GlobalChat] Global chat disabled for you.").build())
	data.clients.connectedClients[context.playerId].setPrefs(prefs)
	data.clients.connectedClients[context.playerId].setPrefs(prefs)
	gcUserPrefs[context.playerId]['toggle'] = False
	savePrefs()

@plugins.commandHook("g")
def chat(context, params):
	if not data.clients.connectedClients[context.playerId].getPrefs()['globalChat']:
		context.sendCryptoPacket(packetFactory.ChatPacket(context.playerId, "[GlobalChat] You do not have global chat enabled, and can not send a global message.").build())
		return
	print("[GlobalChat] <%s> %s" % (data.players.playerList[context.playerId][0], params[3:]))
	for client in data.clients.connectedClients.values():
		if client.getPrefs()['globalChat'] and client.getHandle() is not None:
			client.getHandle().sendCryptoPacket(packetFactory.TeamChatPacket(context.playerId, data.players.playerList[context.playerId][0], "[G] %s" % params[3:]).build())
<|endoftext|>


<commit_before>#! /usr/bin/python
# Written by Dan Mandle http://dan.mandle.me September 2012
# Modified by Broda Noel @brodanoel (in all social networks)
# License: GPL 2.0

from gps import *
from time import *
import time
import threading
import sys

gpsd = None #seting the global variable

class GpsPoller(threading.Thread):
  def __init__(self):
    threading.Thread.__init__(self)
    global gpsd #bring it in scope
    gpsd = gps(mode=WATCH_ENABLE) #starting the stream of info
    self.current_value = None
    self.running = True #setting the thread running to true

  def run(self):
    global gpsd
    while gpsp.running:
      gpsd.next() #this will continue to loop and grab EACH set of gpsd info to clear the buffer

if __name__ == '__main__':
  gpsp = GpsPoller() # create the thread
  try:
    gpsp.start() # start it up
    attempts = 0
    gotData = False

    while not gotData and attempts < 3:
      #It may take a second or two to get good data

      # TODO What it we get a "nan"?
      if gpsd.fix.latitude != 0 or gpsd.fix.longitude != 0:
        gotData = True
        attempts += 1

        output = '{"status": "ok", "data": '

        output += '{'
        output += '"latitude":' + str(gpsd.fix.latitude) + ','
        output += '"longitude":' + str(gpsd.fix.longitude) + ','
        output += '"time":"' + str(gpsd.fix.time) + '",'
        output += '"utcTime":"' + str(gpsd.utc) + '",'
        output += '"altitude":' + str(gpsd.fix.altitude) + ','
        output += '"eps":' + str(gpsd.fix.eps) + ','
        output += '"epx":' + str(gpsd.fix.epx) + ','
        output += '"epv":' + str(gpsd.fix.epv) + ','
        output += '"ept":' + str(gpsd.fix.ept) + ','
        output += '"speed":' + str(gpsd.fix.speed) + ','
        output += '"climb":' + str(gpsd.fix.climb) + ','
        output += '"track":' + str(gpsd.fix.track) + ','
        output += '"mode":' + str(gpsd.fix.mode)
        #output += 'satellites:' + str(gpsd.satellites)
        output += '} }'

        sys.stdout.write(output)

      else:
        time.sleep(1) #set to whatever

    if not gotData:
      sys.stdout.write('{ "status": "error", "message": "Too much attempts" }')

  except:
    sys.stdout.write('{"status": "error", "message": "' + str(sys.exc_info()) + '"}')

gpsp.running = False
gpsp.join() # wait for the thread to finish what it's doing<commit_msg>Can't believe it...<commit_after>#! /usr/bin/python
# Written by Dan Mandle http://dan.mandle.me September 2012
# Modified by Broda Noel @brodanoel (in all social networks)
# License: GPL 2.0

from gps import *
from time import *
import time
import threading
import sys

gpsd = None #seting the global variable

class GpsPoller(threading.Thread):
  def __init__(self):
    threading.Thread.__init__(self)
    global gpsd #bring it in scope
    gpsd = gps(mode=WATCH_ENABLE) #starting the stream of info
    self.current_value = None
    self.running = True #setting the thread running to true

  def run(self):
    global gpsd
    while gpsp.running:
      gpsd.next() #this will continue to loop and grab EACH set of gpsd info to clear the buffer

if __name__ == '__main__':
  gpsp = GpsPoller() # create the thread
  try:
    gpsp.start() # start it up
    attempts = 0
    gotData = False

    while not gotData and attempts < 3:
      attempts += 1

      #It may take a second or two to get good data
      # TODO What it we get a "nan"?
      if gpsd.fix.latitude != 0 or gpsd.fix.longitude != 0:
        gotData = True

        output = '{"status": "ok", "data": '

        output += '{'
        output += '"latitude":' + str(gpsd.fix.latitude) + ','
        output += '"longitude":' + str(gpsd.fix.longitude) + ','
        output += '"time":"' + str(gpsd.fix.time) + '",'
        output += '"utcTime":"' + str(gpsd.utc) + '",'
        output += '"altitude":' + str(gpsd.fix.altitude) + ','
        output += '"eps":' + str(gpsd.fix.eps) + ','
        output += '"epx":' + str(gpsd.fix.epx) + ','
        output += '"epv":' + str(gpsd.fix.epv) + ','
        output += '"ept":' + str(gpsd.fix.ept) + ','
        output += '"speed":' + str(gpsd.fix.speed) + ','
        output += '"climb":' + str(gpsd.fix.climb) + ','
        output += '"track":' + str(gpsd.fix.track) + ','
        output += '"mode":' + str(gpsd.fix.mode)
        #output += 'satellites:' + str(gpsd.satellites)
        output += '} }'

        sys.stdout.write(output)

      else:
        time.sleep(1) #set to whatever

    if not gotData:
      sys.stdout.write('{ "status": "error", "message": "Too much attempts" }')

  except:
    sys.stdout.write('{"status": "error", "message": "' + str(sys.exc_info()) + '"}')

gpsp.running = False
gpsp.join() # wait for the thread to finish what it's doing<|endoftext|>


<commit_before>from optparse import OptionParser
arg_parser = OptionParser()
arg_parser.add_option("-a", "--agency", dest="agency", action="store", type="string", default=None, help="Specify an agency to which to limit the dump.")
arg_parser.add_option("-d", "--docket", dest="docket", action="store", type="string", default=None, help="Specify a docket to which to limit the dump.")

def run(options, args):
    import urllib2, httplib
    import settings
    import os, time, sys
    from regsdotgov.search import search, parsed_search
    from regs_common.transfer import download

    search_args = {}
    id_string = 'all'
    if options.agency and options.docket:
        raise Exception("Specify either an agency or a docket")
    elif options.agency:
        search_args = {'agency': options.agency}
        id_string = 'agency_' + options.agency
    elif options.docket:
        search_args = {'docket': options.docket}
        id_string = 'docket_' + options.docket.replace('-', '_')

    # delete old dumps
    [os.unlink(os.path.join(settings.DUMP_DIR, file)) for file in os.listdir(settings.DUMP_DIR) if file.startswith('dump_%s' % id_string) and file.endswith('.json')]

    # keep stats
    stats = {'downloaded': 0, 'failed': 0}

    # start new dumps
    position = 0
    total = parsed_search(1, 0, **search_args)['totalNumRecords']
    num_digits = len(str(settings.DUMP_END))
    while position <= total:
        for i in range(3):
            try:
                current_str = (position / settings.DUMP_INCREMENT) + 1
                total_str = '?' if total == 1 else (total / settings.DUMP_INCREMENT) + 1
                print "Downloading page %s of %s..." % (current_str, total_str)
                download(
                    search(settings.DUMP_INCREMENT, position, **search_args),
                    os.path.join(settings.DUMP_DIR, 'dump_%s_%s.json' % (id_string, str(position).zfill(num_digits))),
                )
                stats['downloaded'] += 1
                break
            except (urllib2.HTTPError, httplib.HTTPException) as e:
                if i < 2:
                    if hasattr(e, 'code') and e.code in (503, 429) and 'rate' in e.read().lower():
                        print 'Download failed because of rate limiting; will retry in an hour...'
                        time.sleep(3600)
                    else:
                        print 'Download failed; will retry in 10 seconds...'
                        time.sleep(10)
                else:
                    print 'System troubles; giving up.'
                    raise

        position += settings.DUMP_INCREMENT

    return stats
<commit_msg>Switch to fixed 1000 per page because of new limits.<commit_after>from optparse import OptionParser
arg_parser = OptionParser()
arg_parser.add_option("-a", "--agency", dest="agency", action="store", type="string", default=None, help="Specify an agency to which to limit the dump.")
arg_parser.add_option("-d", "--docket", dest="docket", action="store", type="string", default=None, help="Specify a docket to which to limit the dump.")

def run(options, args):
    import urllib2, httplib
    import settings
    import os, time, sys
    from regsdotgov.search import search, parsed_search
    from regs_common.transfer import download

    search_args = {}
    id_string = 'all'
    if options.agency and options.docket:
        raise Exception("Specify either an agency or a docket")
    elif options.agency:
        search_args = {'agency': options.agency}
        id_string = 'agency_' + options.agency
    elif options.docket:
        search_args = {'docket': options.docket}
        id_string = 'docket_' + options.docket.replace('-', '_')

    # delete old dumps
    [os.unlink(os.path.join(settings.DUMP_DIR, file)) for file in os.listdir(settings.DUMP_DIR) if file.startswith('dump_%s' % id_string) and file.endswith('.json')]

    # keep stats
    stats = {'downloaded': 0, 'failed': 0}

    # start new dumps
    position = 0
    increment = 1000
    total = parsed_search(1, 0, **search_args)['totalNumRecords']
    num_digits = len(str(settings.DUMP_END))
    while position <= total:
        for i in range(3):
            try:
                current_str = (position / increment) + 1
                total_str = '?' if total == 1 else (total / increment) + 1
                print "Downloading page %s of %s..." % (current_str, total_str)
                download(
                    search(increment, position, **search_args),
                    os.path.join(settings.DUMP_DIR, 'dump_%s_%s.json' % (id_string, str(position).zfill(num_digits))),
                )
                stats['downloaded'] += 1
                break
            except (urllib2.HTTPError, httplib.HTTPException) as e:
                if i < 2:
                    if hasattr(e, 'code') and e.code in (503, 429) and 'rate' in e.read().lower():
                        print 'Download failed because of rate limiting; will retry in an hour...'
                        time.sleep(3600)
                    else:
                        print 'Download failed; will retry in 10 seconds...'
                        time.sleep(10)
                else:
                    print 'System troubles; giving up.'
                    raise

        position += increment

    return stats
<|endoftext|>


<commit_before># Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Host Filter is a driver mechanism for requesting instance resources.
Three drivers are included: AllHosts, Flavor & JSON. AllHosts just
returns the full, unfiltered list of hosts. Flavor is a hard coded
matching mechanism based on flavor criteria and JSON is an ad-hoc
filter grammar.

Why JSON? The requests for instances may come in through the
REST interface from a user or a parent Zone.
Currently Flavors and/or InstanceTypes are used for
specifing the type of instance desired. Specific Nova users have
noted a need for a more expressive way of specifying instances.
Since we don't want to get into building full DSL this is a simple
form as an example of how this could be done. In reality, most
consumers will use the more rigid filters such as FlavorFilter.

Note: These are hard filters. All capabilities used must be present
or the host will be excluded. If you want soft filters use the weighting
mechanism which is intended for the more touchy-feely capabilities.
"""

import json

from nova import exception
from nova import flags
from nova import log as logging
from nova import utils

LOG = logging.getLogger('nova.scheduler.host_filter')

FLAGS = flags.FLAGS
flags.DEFINE_string('default_host_filter_driver',
                    'nova.scheduler.host_filter.AllHostsFilter',
                    'Which driver to use for filtering hosts.')

class HostFilter(object):
    """Base class for host filter drivers."""

    def instance_type_to_filter(self, instance_type):
        """Convert instance_type into a filter for most common use-case."""
        raise NotImplementedError()

    def filter_hosts(self, zone_manager, query):
        """Return a list of hosts that fulfill the filter."""
        raise NotImplementedError()

    def _full_name(self):
        """module.classname of the filter driver"""
        return "%s.%s" % (self.__module__, self.__class__.__name__)

class AllHostsFilter(HostFilter):
    """NOP host filter driver. Returns all hosts in ZoneManager.
    This essentially does what the old Scheduler+Chance used
    to give us."""

    def instance_type_to_filter(self, instance_type):
        """Return anything to prevent base-class from raising
        exception."""
        return (self._full_name(), instance_type)

    def filter_hosts(self, zone_manager, query):
        """Return a list of hosts from ZoneManager list."""
        return [(host, services)
               for host, services in zone_manager.service_states.iteritems()]

class FlavorFilter(HostFilter):
    """HostFilter driver hard-coded to work with flavors."""

    def instance_type_to_filter(self, instance_type):
        """Use instance_type to filter hosts."""
        return (self._full_name(), instance_type)

    def filter_hosts(self, zone_manager, query):
        """Return a list of hosts that can create instance_type."""
        instance_type = query
        selected_hosts = []
        for host, services in zone_manager.service_states.iteritems():
            capabilities = services.get('compute', {})
            host_ram_mb = capabilities['host_memory']['free']
            disk_bytes = capabilities['disk']['available']
            if host_ram_mb >= instance_type['memory_mb'] and \
                disk_bytes >= instance_type['local_gb']:
                    selected_hosts.append((host, capabilities))
        return selected_hosts

#host entries (currently) are like:
#    {'host_name-description': 'Default install of XenServer',
#    'host_hostname': 'xs-mini',
#    'host_memory': {'total': 8244539392,
#        'overhead': 184225792,
#        'free': 3868327936,
#        'free-computed': 3840843776},
#    'host_other-config': {},
#    'host_ip_address': '192.168.1.109',
#    'host_cpu_info': {},
#    'disk': {'available': 32954957824,
#        'total': 50394562560,
#        'used': 17439604736},
#    'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f',
#    'host_name-label': 'xs-mini'}

# instance_type table has:
#name = Column(String(255), unique=True)
#memory_mb = Column(Integer)
#vcpus = Column(Integer)
#local_gb = Column(Integer)
#flavorid = Column(Integer, unique=True)
#swap = Column(Integer, nullable=False, default=0)
#rxtx_quota = Column(Integer, nullable=False, default=0)
#rxtx_cap = Column(Integer, nullable=False, default=0)

class JsonFilter(HostFilter):
    """Host Filter driver to allow simple JSON-based grammar for
       selecting hosts."""

    def _equals(self, args):
        """First term is == all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs != rhs:
                return False
        return True

    def _less_than(self, args):
        """First term is < all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs >= rhs:
                return False
        return True

    def _greater_than(self, args):
        """First term is > all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs <= rhs:
                return False
        return True

    def _in(self, args):
        """First term is in set of remaining terms"""
        if len(args) < 2:
            return False
        return args[0] in args[1:]

    def _less_than_equal(self, args):
        """First term is <= all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs > rhs:
                return False
        return True

    def _greater_than_equal(self, args):
        """First term is >= all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs < rhs:
                return False
        return True

    def _not(self, args):
        """Flip each of the arguments."""
        if len(args) == 0:
            return False
        return [not arg for arg in args]

    def _or(self, args):
        """True if any arg is True."""
        return True in args

    def _and(self, args):
        """True if all args are True."""
        return False not in args

    commands = {
        '=': _equals,
        '<': _less_than,
        '>': _greater_than,
        'in': _in,
        '<=': _less_than_equal,
        '>=': _greater_than_equal,
        'not': _not,
        'or': _or,
        'and': _and,
    }

    def instance_type_to_filter(self, instance_type):
        """Convert instance_type into JSON filter object."""
        required_ram = instance_type['memory_mb']
        required_disk = instance_type['local_gb']
        query = ['and',
                    ['>=', '$compute.host_memory.free', required_ram],
                    ['>=', '$compute.disk.available', required_disk]
                ]
        return (self._full_name(), json.dumps(query))

    def _parse_string(self, string, host, services):
        """Strings prefixed with $ are capability lookups in the
        form '$service.capability[.subcap*]'"""
        if not string:
            return None
        if string[0] != '$':
            return string

        path = string[1:].split('.')
        for item in path:
            services = services.get(item, None)
            if not services:
                return None
        return services

    def _process_filter(self, zone_manager, query, host, services):
        """Recursively parse the query structure."""
        if len(query) == 0:
            return True
        cmd = query[0]
        method = self.commands[cmd]  # Let exception fly.
        cooked_args = []
        for arg in query[1:]:
            if isinstance(arg, list):
                arg = self._process_filter(zone_manager, arg, host, services)
            elif isinstance(arg, basestring):
                arg = self._parse_string(arg, host, services)
            if arg != None:
                cooked_args.append(arg)
        result = method(self, cooked_args)
        return result

    def filter_hosts(self, zone_manager, query):
        """Return a list of hosts that can fulfill filter."""
        expanded = json.loads(query)
        hosts = []
        for host, services in zone_manager.service_states.iteritems():
            r = self._process_filter(zone_manager, expanded, host, services)
            if isinstance(r, list):
                r = True in r
            if r:
                hosts.append((host, services))
        return hosts

DRIVERS = [AllHostsFilter, FlavorFilter, JsonFilter]

def choose_driver(driver_name=None):
    """Since the caller may specify which driver to use we need
       to have an authoritative list of what is permissible. This
       function checks the driver name against a predefined set
       of acceptable drivers."""

    if not driver_name:
        driver_name = FLAGS.default_host_filter_driver
    for driver in DRIVERS:
        if "%s.%s" % (driver.__module__, driver.__name__) == driver_name:
            return driver()
    raise exception.SchedulerHostFilterDriverNotFound(driver_name=driver_name)
<commit_msg>revised file docs<commit_after># Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Host Filter is a driver mechanism for requesting instance resources.
Three drivers are included: AllHosts, Flavor & JSON. AllHosts just
returns the full, unfiltered list of hosts. Flavor is a hard coded
matching mechanism based on flavor criteria and JSON is an ad-hoc
filter grammar.

Why JSON? The requests for instances may come in through the
REST interface from a user or a parent Zone.
Currently Flavors and/or InstanceTypes are used for
specifing the type of instance desired. Specific Nova users have
noted a need for a more expressive way of specifying instances.
Since we don't want to get into building full DSL this is a simple
form as an example of how this could be done. In reality, most
consumers will use the more rigid filters such as FlavorFilter.

Note: These are "required" capability filters. These capabilities
used must be present or the host will be excluded. The hosts
returned are then weighed by the Weighted Scheduler. Weights
can take the more esoteric factors into consideration (such as
server affinity and customer separation).
"""

import json

from nova import exception
from nova import flags
from nova import log as logging
from nova import utils

LOG = logging.getLogger('nova.scheduler.host_filter')

FLAGS = flags.FLAGS
flags.DEFINE_string('default_host_filter_driver',
                    'nova.scheduler.host_filter.AllHostsFilter',
                    'Which driver to use for filtering hosts.')

class HostFilter(object):
    """Base class for host filter drivers."""

    def instance_type_to_filter(self, instance_type):
        """Convert instance_type into a filter for most common use-case."""
        raise NotImplementedError()

    def filter_hosts(self, zone_manager, query):
        """Return a list of hosts that fulfill the filter."""
        raise NotImplementedError()

    def _full_name(self):
        """module.classname of the filter driver"""
        return "%s.%s" % (self.__module__, self.__class__.__name__)

class AllHostsFilter(HostFilter):
    """NOP host filter driver. Returns all hosts in ZoneManager.
    This essentially does what the old Scheduler+Chance used
    to give us."""

    def instance_type_to_filter(self, instance_type):
        """Return anything to prevent base-class from raising
        exception."""
        return (self._full_name(), instance_type)

    def filter_hosts(self, zone_manager, query):
        """Return a list of hosts from ZoneManager list."""
        return [(host, services)
               for host, services in zone_manager.service_states.iteritems()]

class FlavorFilter(HostFilter):
    """HostFilter driver hard-coded to work with flavors."""

    def instance_type_to_filter(self, instance_type):
        """Use instance_type to filter hosts."""
        return (self._full_name(), instance_type)

    def filter_hosts(self, zone_manager, query):
        """Return a list of hosts that can create instance_type."""
        instance_type = query
        selected_hosts = []
        for host, services in zone_manager.service_states.iteritems():
            capabilities = services.get('compute', {})
            host_ram_mb = capabilities['host_memory']['free']
            disk_bytes = capabilities['disk']['available']
            if host_ram_mb >= instance_type['memory_mb'] and \
                disk_bytes >= instance_type['local_gb']:
                    selected_hosts.append((host, capabilities))
        return selected_hosts

#host entries (currently) are like:
#    {'host_name-description': 'Default install of XenServer',
#    'host_hostname': 'xs-mini',
#    'host_memory': {'total': 8244539392,
#        'overhead': 184225792,
#        'free': 3868327936,
#        'free-computed': 3840843776},
#    'host_other-config': {},
#    'host_ip_address': '192.168.1.109',
#    'host_cpu_info': {},
#    'disk': {'available': 32954957824,
#        'total': 50394562560,
#        'used': 17439604736},
#    'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f',
#    'host_name-label': 'xs-mini'}

# instance_type table has:
#name = Column(String(255), unique=True)
#memory_mb = Column(Integer)
#vcpus = Column(Integer)
#local_gb = Column(Integer)
#flavorid = Column(Integer, unique=True)
#swap = Column(Integer, nullable=False, default=0)
#rxtx_quota = Column(Integer, nullable=False, default=0)
#rxtx_cap = Column(Integer, nullable=False, default=0)

class JsonFilter(HostFilter):
    """Host Filter driver to allow simple JSON-based grammar for
       selecting hosts."""

    def _equals(self, args):
        """First term is == all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs != rhs:
                return False
        return True

    def _less_than(self, args):
        """First term is < all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs >= rhs:
                return False
        return True

    def _greater_than(self, args):
        """First term is > all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs <= rhs:
                return False
        return True

    def _in(self, args):
        """First term is in set of remaining terms"""
        if len(args) < 2:
            return False
        return args[0] in args[1:]

    def _less_than_equal(self, args):
        """First term is <= all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs > rhs:
                return False
        return True

    def _greater_than_equal(self, args):
        """First term is >= all the other terms."""
        if len(args) < 2:
            return False
        lhs = args[0]
        for rhs in args[1:]:
            if lhs < rhs:
                return False
        return True

    def _not(self, args):
        """Flip each of the arguments."""
        if len(args) == 0:
            return False
        return [not arg for arg in args]

    def _or(self, args):
        """True if any arg is True."""
        return True in args

    def _and(self, args):
        """True if all args are True."""
        return False not in args

    commands = {
        '=': _equals,
        '<': _less_than,
        '>': _greater_than,
        'in': _in,
        '<=': _less_than_equal,
        '>=': _greater_than_equal,
        'not': _not,
        'or': _or,
        'and': _and,
    }

    def instance_type_to_filter(self, instance_type):
        """Convert instance_type into JSON filter object."""
        required_ram = instance_type['memory_mb']
        required_disk = instance_type['local_gb']
        query = ['and',
                    ['>=', '$compute.host_memory.free', required_ram],
                    ['>=', '$compute.disk.available', required_disk]
                ]
        return (self._full_name(), json.dumps(query))

    def _parse_string(self, string, host, services):
        """Strings prefixed with $ are capability lookups in the
        form '$service.capability[.subcap*]'"""
        if not string:
            return None
        if string[0] != '$':
            return string

        path = string[1:].split('.')
        for item in path:
            services = services.get(item, None)
            if not services:
                return None
        return services

    def _process_filter(self, zone_manager, query, host, services):
        """Recursively parse the query structure."""
        if len(query) == 0:
            return True
        cmd = query[0]
        method = self.commands[cmd]  # Let exception fly.
        cooked_args = []
        for arg in query[1:]:
            if isinstance(arg, list):
                arg = self._process_filter(zone_manager, arg, host, services)
            elif isinstance(arg, basestring):
                arg = self._parse_string(arg, host, services)
            if arg != None:
                cooked_args.append(arg)
        result = method(self, cooked_args)
        return result

    def filter_hosts(self, zone_manager, query):
        """Return a list of hosts that can fulfill filter."""
        expanded = json.loads(query)
        hosts = []
        for host, services in zone_manager.service_states.iteritems():
            r = self._process_filter(zone_manager, expanded, host, services)
            if isinstance(r, list):
                r = True in r
            if r:
                hosts.append((host, services))
        return hosts

DRIVERS = [AllHostsFilter, FlavorFilter, JsonFilter]

def choose_driver(driver_name=None):
    """Since the caller may specify which driver to use we need
       to have an authoritative list of what is permissible. This
       function checks the driver name against a predefined set
       of acceptable drivers."""

    if not driver_name:
        driver_name = FLAGS.default_host_filter_driver
    for driver in DRIVERS:
        if "%s.%s" % (driver.__module__, driver.__name__) == driver_name:
            return driver()
    raise exception.SchedulerHostFilterDriverNotFound(driver_name=driver_name)
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010 Citrix Systems, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
#    Copyright (c) 2010 Citrix Systems, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Stubouts, mocks and fixtures for the test suite"""
<commit_msg>fake session clean-up<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (c) 2010 Citrix Systems, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Stubouts, mocks and fixtures for the test suite"""
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Bind LLVM functions to ctypes.
Originally adapated from Bitey, blaze/blir/bind.py.
"""

from __future__ import print_function, division, absolute_import

from llvm.core import Module
import llvm.core
import llvm.ee
import ctypes
import io
import os
import sys

PY3 = sys.version_info[0] >= 3

def map_llvm_to_ctypes(llvm_type, py_module=None):
    '''
    Map an LLVM type to an equivalent ctypes type. py_module is an
    optional module that is used for structure wrapping.  If
    structures are found, the struct definitions will be created in
    that module.
    '''
    kind = llvm_type.kind
    if kind == llvm.core.TYPE_INTEGER:
        ctype = getattr(ctypes,"c_int"+str(llvm_type.width))

    elif kind == llvm.core.TYPE_DOUBLE:
        ctype = ctypes.c_double

    elif kind == llvm.core.TYPE_FLOAT:
        ctype = ctypes.c_float

    elif kind == llvm.core.TYPE_VOID:
        ctype = None

    elif kind == llvm.core.TYPE_POINTER:
        pointee = llvm_type.pointee
        p_kind = pointee.kind
        if p_kind == llvm.core.TYPE_INTEGER:
            width = pointee.width

            # Special case:  char * is mapped to strings
            if width == 8:
                ctype = ctypes.c_char_p
            else:
                ctype = ctypes.POINTER(map_llvm_to_ctypes(pointee, py_module))

        # Special case: void * mapped to c_void_p type
        elif p_kind == llvm.core.TYPE_VOID:
            ctype = ctypes.c_void_p
        else:
            ctype = ctypes.POINTER(map_llvm_to_ctypes(pointee, py_module))

    elif kind == llvm.core.TYPE_STRUCT:
        struct_name = llvm_type.name.split('.')[-1]
        if not PY3:
            struct_name = struct_name.encode('ascii')

        # If the named type is already known, return it
        if py_module:
            struct_type = getattr(py_module, struct_name, None)
        else:
            struct_type = None

        if struct_type and issubclass(struct_type, ctypes.Structure):
            return struct_type

        # If there is an object with the name of the structure already present and it has
        # the field names specified, use those names to help out
        if hasattr(struct_type, '_fields_'):
            names = struct_type._fields_
        else:
            names = [ "e"+str(n) for n in range(llvm_type.element_count) ]

        # Create a class definition for the type. It is critical that this
        # Take place before the handling of members to avoid issues with
        # self-referential data structures
        ctype = type(ctypes.Structure)(struct_name, (ctypes.Structure,),
                                       { '__module__' : py_module.__name__ })

        if py_module:
            setattr(py_module, struct_name, ctype)

        # Resolve the structure fields
        fields = [ (name, map_llvm_to_ctypes(elem, py_module))
                   for name, elem in zip(names, llvm_type.elements) ]

        # Set the fields member of the type last.  The order is critical
        # to deal with self-referential structures.
        setattr(ctype, '_fields_', fields)
    else:
        raise TypeError("Unknown type %s" % llvm_type)

    return ctype

def wrap_llvm_function(func, engine, py_module):
    '''
    Create a ctypes wrapper around an LLVM function.
    engine is the LLVM execution engine.
    func is an LLVM function instance.
    py_module is a Python module where to put the wrappers.
    '''
    args = func.type.pointee.args
    ret_type = func.type.pointee.return_type
    try:
        ret_ctype = map_llvm_to_ctypes(ret_type, py_module)
        args_ctypes = [map_llvm_to_ctypes(arg, py_module) for arg in args]
    except TypeError as e:
        if 'BITEYDEBUG' in os.environ:
            print("Couldn't wrap %s. Reason %s" % (func.name, e))
        return None

    # Declare the ctypes function prototype
    functype = ctypes.CFUNCTYPE(ret_ctype, *args_ctypes)

    # Get the function point from the execution engine
    addr = engine.get_pointer_to_function(func)

    # Make a ctypes callable out of it
    wrapper = functype(addr)

    # Set it in the module
    setattr(py_module, func.name, wrapper)
    wrapper.__name__ = func.name

def wrap_llvm_module(llvm_module, engine, py_module):
    '''
    Build ctypes wrappers around an existing LLVM module and execution
    engine.  py_module is an existing Python module that will be
    populated with the resulting wrappers.
    '''
    functions = [func for func in llvm_module.functions
                 if not func.name.startswith("_")
                 and not func.is_declaration
                 and func.linkage == llvm.core.LINKAGE_EXTERNAL]
    for func in functions:
        wrap_llvm_function(func, engine, py_module)

def wrap_llvm_bitcode(bitcode, py_module):
    '''
    Given a byte-string of LLVM bitcode and a Python module,
    populate the module with ctypes bindings for public methods
    in the bitcode.
    '''
    llvm_module = Module.from_bitcode(io.BytesIO(bitcode))
    engine = llvm.ee.ExecutionEngine.new(llvm_module)
    wrap_llvm_module(llvm_module, engine, py_module)
    setattr(py_module, '_llvm_module', llvm_module)
    setattr(py_module, '_llvm_engine', engine)
<commit_msg>Add some tests to math support<commit_after><|endoftext|>


<commit_before>import __builtin__
import math

import numpy as np
# from numpy.ctypeslib import _typecodes

import numba
from numba import  error
from numba.minivect.minitypes import *
from numba.minivect.minitypes import map_dtype
from numba.minivect import minitypes, minierror

#------------------------------------------------------------------------
# Numba's extension of the minivect type system
#------------------------------------------------------------------------

# Patch repr of objects to print "object_" instead of "PyObject *"
minitypes.ObjectType.__repr__ = lambda self: "object_"

class NumbaType(minitypes.Type):
    is_numba_type = True

class NumbaKeyHashingType(minitypes.KeyHashingType):
    is_numba_type = True

#------------------------------------------------------------------------
# Python Types
#------------------------------------------------------------------------

class ContainerListType(NumbaKeyHashingType, minitypes.ObjectType):

    def __init__(self, dtype, size=-1):
        super(ContainerListType, self).__init__()
        self.dtype = dtype
        self.size = size

    @property
    def key(self):
        return self.dtype, self.size

    def __repr__(self):
        return "%s(%s, %s)" % (self.name, self.dtype, self.size)

class TupleType(ContainerListType):

    is_tuple = True
    name = "tuple"

class ListType(ContainerListType):
    is_list = True
    name = "list"

class DictType(NumbaType, minitypes.ObjectType):
    is_dict = True
    name = "dict"

    def __str__(self):
        return "dict(%s)" % ", ".join(["..."] * self.size)

class IteratorType(NumbaType, minitypes.ObjectType):
    is_iterator = True
    subtypes = ['base_type']

    def __init__(self, base_type, **kwds):
        super(IteratorType, self).__init__(**kwds)
        self.base_type = base_type

    def __repr__(self):
        return "iterator<%s>" % (self.base_type,)

class ModuleType(NumbaType, minitypes.ObjectType):
    """
    Represents a type for modules.

    Attributes:
        is_numpy_module: whether the module is the numpy module
        module: in case of numpy, the numpy module or a submodule
    """

    is_module = True
    is_numpy_module = False

    def __init__(self, module, **kwds):
        super(ModuleType, self).__init__(**kwds)
        self.module = module
        self.is_numpy_module = module is np
        self.is_numba_module = module is numba
        self.is_math_module = module is math

    def __repr__(self):
        if self.is_numpy_module:
            return 'numpy'
        else:
            return 'ModuleType'

    @property
    def comparison_type_list(self):
        return (self.module,)

class ModuleAttributeType(NumbaKeyHashingType, minitypes.ObjectType):
    is_module_attribute = True

    module = None
    attr = None

    def __init__(self, **kwds):
        super(ModuleAttributeType, self).__init__(**kwds)
        base_name, dot, rest = self.module.__name__.partition(".")
        self.is_numpy_attribute = base_name == "numpy"

    def __repr__(self):
        return "%s.%s" % (self.module.__name__, self.attr)

    @property
    def value(self):
        return getattr(self.module, self.attr)

    @property
    def key(self):
        return (self.module, self.attr)

class NumpyAttributeType(ModuleAttributeType):
    """
    Type for attributes of a numpy (sub)module.

    Attributes:
        module: the numpy (sub)module
        attr: the attribute name (str)
    """

    is_numpy_attribute = True

class MethodType(NumbaType, minitypes.ObjectType):
    """
    Method of something.

        base_type: the object type the attribute was accessed on
    """

    is_method = True

    def __init__(self, base_type, attr_name, **kwds):
        super(MethodType, self).__init__(**kwds)
        self.base_type = base_type
        self.attr_name = attr_name

class NumpyDtypeType(NumbaType, minitypes.ObjectType):
    is_numpy_dtype = True
    dtype = None # NumPy dtype type

    def resolve(self):
        return map_dtype(self.dtype)

    def __repr__(self):
        return "NumpyDtype(%s)" % self.resolve()

class ResolvedNumpyDtypeType(NumbaType, minitypes.ObjectType):
    is_numpy_dtype = True
    dtype_type = None # numba dtype type

    def resolve(self):
        return self.dtype_type

    def __repr__(self):
        return "NumpyDtype(%s)" % self.resolve()

class TBAAType(NumbaType):
    """
    Type based alias analysis type. See numba/metadata.py.
    """

    is_tbaa = True

    def __init__(self, name, root, **kwds):
        super(TBAAType, self).__init__(**kwds)
        self.name = name
        self.root = root

    @property
    def comparison_type_list(self):
        return [self.name, self.root]

    def __repr__(self):
        return "tbaa(%s)" % self.name

class EllipsisType(NumbaType, minitypes.ObjectType):
    is_ellipsis = True

    def __eq__(self, other):
        return other.is_ellipsis

    def __repr__(self):
        return "..."

class SliceType(NumbaType, minitypes.ObjectType):
    is_slice = True

    def __eq__(self, other):
        return other.is_slice

    def __repr__(self):
        return ":"

class NewAxisType(NumbaType, minitypes.ObjectType):
    is_newaxis = True

    def __eq__(self, other):
        return other.is_newaxis

    def __repr__(self):
        return "newaxis"

class GlobalType(NumbaType, minitypes.ObjectType):
    is_global = True
    name = None

    def __repr__(self):
        return "global(%s)" % self.name

class BuiltinType(NumbaType, minitypes.ObjectType):
    is_builtin = True

    def __init__(self, name, **kwds):
        super(BuiltinType, self).__init__(**kwds)
        self.name = name
        self.func = getattr(__builtin__, name)

    def __repr__(self):
        return "builtin(%s)" % self.name

class RangeType(NumbaType, minitypes.ObjectType):
    is_range = True

    def __repr__(self):
        return "range(...)"

class NoneType(NumbaType, minitypes.ObjectType):
    is_none = True

    def __repr__(self):
        return "<type(None)>"

#------------------------------------------------------------------------
# Function Types
#------------------------------------------------------------------------

class CTypesFunctionType(NumbaType, minitypes.ObjectType):
    is_ctypes_function = True

    def __init__(self, ctypes_func, restype, argtypes, **kwds):
        super(CTypesFunctionType, self).__init__(**kwds)
        self.ctypes_func = ctypes_func
        self.signature = minitypes.FunctionType(return_type=restype,
                                                args=argtypes)

    def __repr__(self):
        return "<ctypes function %s>" % (self.signature,)

class AutojitType(NumbaType, minitypes.ObjectType):
    """
    Type for autojitting functions.
    """

    is_autojit_function = True

    def __init__(self, autojit_func, **kwds):
        super(AutojitType, self).__init__(**kwds)
        self.autojit_func = autojit_func

    def __repr__(self):
        return "<autojit(%s)>" % self.autojit_func

class JitType(NumbaType, minitypes.ObjectType):
    """
    Type for autojitting functions.
    """

    is_jit_function = True

    def __init__(self, jit_func, **kwds):
        super(JitType, self).__init__(**kwds)
        self.jit_func = jit_func

    def __repr__(self):
        return "<jit(%s)>" % self.jit_func

#------------------------------------------------------------------------
# Pointer Types
#------------------------------------------------------------------------

class NULLType(NumbaType):
    """
    Null pointer type that can be compared or assigned to any other
    pointer type.
    """

    is_null = True

    def __repr__(self):
        return "<type(NULL)>"

class CTypesPointerType(NumbaType):
    def __init__(self, pointer_type, address, **kwds):
        super(CTypesPointer, self).__init__(**kwds)
        self.pointer_type = pointer_type
        self.address = address

class SizedPointerType(NumbaType, minitypes.PointerType):
    """
    A pointer with knowledge of its range.

    E.g. an array's 'shape' or 'strides' attribute.
    This also allow tuple unpacking.
    """

    size = None
    is_sized_pointer = True

    def __repr__(self):
        return "%r<%s>" % (self.base_type.pointer(), self.size)

    def __eq__(self, other):
        if other.is_sized_pointer:
            return (self.base_type == other.base_type and
                    self.size == other.size)
        return other.is_pointer and self.base_type == other.base_type

    def __hash__(self):
        return hash(self.base_type.pointer())

class CastType(NumbaType, minitypes.ObjectType):
    """
    A type instance in user code. e.g. double(value). The Name node will have
    a cast-type with dst_type 'double'.
    """

    is_cast = True

    subtypes = ['dst_type']

    def __init__(self, dst_type, **kwds):
        super(CastType, self).__init__(**kwds)
        self.dst_type = dst_type

    def __repr__(self):
        return "<cast(%s)>" % self.dst_type

#------------------------------------------------------------------------
# Aggregate Types
#------------------------------------------------------------------------

class struct(minitypes.struct):
    __doc__ = minitypes.struct.__doc__

    def ref(self):
        return ReferenceType(self)

#    def __repr__(self):
#        return "struct(...)"

#------------------------------------------------------------------------
# References
#------------------------------------------------------------------------

class ReferenceType(NumbaType):
    """
    A reference to an (primitive or Python) object. This is passed as a
    pointer and dereferences automatically.

    Currently only supported for structs.
    """

    is_reference = True

    subtypes = ['referenced_type']

    def __init__(self, referenced_type, **kwds):
        super(ReferenceType, self).__init__(**kwds)
        self.referenced_type = referenced_type

    def to_llvm(self, context):
        return self.referenced_type.pointer().to_llvm(context)

    def __repr__(self):
        return "%r.ref()" % (self.referenced_type,)

#------------------------------------------------------------------------
# END OF TYPE DEFINITIONS
#------------------------------------------------------------------------

tuple_ = TupleType(object_, size=0)
none = NoneType()
null_type = NULLType()
intp = minitypes.npy_intp

const_qualifiers = frozenset(["const"])

numpy_array = TBAAType("numpy array", root=object_)
numpy_shape = TBAAType("numpy shape", root=intp.pointer(),
                       qualifiers=const_qualifiers)
numpy_strides = TBAAType("numpy strides", root=intp.pointer(),
                         qualifiers=const_qualifiers)
numpy_ndim = TBAAType("numpy flags", root=int_.pointer())
numpy_dtype = TBAAType("numpy dtype", root=object_)
numpy_base = TBAAType("numpy base", root=object_)
numpy_flags = TBAAType("numpy flags", root=int_.pointer())

iteration_target_type = TBAAType("iteration target", root=char.pointer())
unique_tbaa_type = TBAAType("unique", root=char.pointer())

if __name__ == '__main__':
    import doctest
    doctest.testmod()
<commit_msg>Update the container types<commit_after>import __builtin__
import math

import numpy as np
# from numpy.ctypeslib import _typecodes

import numba
from numba import  error
from numba.minivect.minitypes import *
from numba.minivect.minitypes import map_dtype
from numba.minivect import minitypes, minierror

#------------------------------------------------------------------------
# Numba's extension of the minivect type system
#------------------------------------------------------------------------

# Patch repr of objects to print "object_" instead of "PyObject *"
minitypes.ObjectType.__repr__ = lambda self: "object_"

class NumbaType(minitypes.Type):
    is_numba_type = True

class NumbaKeyHashingType(minitypes.KeyHashingType):
    is_numba_type = True

#------------------------------------------------------------------------
# Python Types
#------------------------------------------------------------------------

class ContainerListType(NumbaKeyHashingType, minitypes.ObjectType):

    subtypes = ['base_type']

    def __init__(self, base_type, size=-1):
        super(ContainerListType, self).__init__()
        self.base_type = base_type
        self.size = size

    @property
    def key(self):
        return self.base_type, self.size

    def is_sized(self):
        return self.size >= 0

    def __repr__(self):
        return "%s(%s, %s)" % (self.name, self.base_type, self.size)

class TupleType(ContainerListType):

    is_tuple = True
    name = "tuple"

class ListType(ContainerListType):
    is_list = True
    name = "list"

class DictType(NumbaType, minitypes.ObjectType):
    is_dict = True
    name = "dict"

    def __str__(self):
        return "dict(%s)" % ", ".join(["..."] * self.size)

class IteratorType(NumbaType, minitypes.ObjectType):
    is_iterator = True
    subtypes = ['base_type']

    def __init__(self, base_type, **kwds):
        super(IteratorType, self).__init__(**kwds)
        self.base_type = base_type

    def __repr__(self):
        return "iterator<%s>" % (self.base_type,)

class ModuleType(NumbaType, minitypes.ObjectType):
    """
    Represents a type for modules.

    Attributes:
        is_numpy_module: whether the module is the numpy module
        module: in case of numpy, the numpy module or a submodule
    """

    is_module = True
    is_numpy_module = False

    def __init__(self, module, **kwds):
        super(ModuleType, self).__init__(**kwds)
        self.module = module
        self.is_numpy_module = module is np
        self.is_numba_module = module is numba
        self.is_math_module = module is math

    def __repr__(self):
        if self.is_numpy_module:
            return 'numpy'
        else:
            return 'ModuleType'

    @property
    def comparison_type_list(self):
        return (self.module,)

class ModuleAttributeType(NumbaKeyHashingType, minitypes.ObjectType):
    is_module_attribute = True

    module = None
    attr = None

    def __init__(self, **kwds):
        super(ModuleAttributeType, self).__init__(**kwds)
        base_name, dot, rest = self.module.__name__.partition(".")
        self.is_numpy_attribute = base_name == "numpy"

    def __repr__(self):
        return "%s.%s" % (self.module.__name__, self.attr)

    @property
    def value(self):
        return getattr(self.module, self.attr)

    @property
    def key(self):
        return (self.module, self.attr)

class NumpyAttributeType(ModuleAttributeType):
    """
    Type for attributes of a numpy (sub)module.

    Attributes:
        module: the numpy (sub)module
        attr: the attribute name (str)
    """

    is_numpy_attribute = True

class MethodType(NumbaType, minitypes.ObjectType):
    """
    Method of something.

        base_type: the object type the attribute was accessed on
    """

    is_method = True

    def __init__(self, base_type, attr_name, **kwds):
        super(MethodType, self).__init__(**kwds)
        self.base_type = base_type
        self.attr_name = attr_name

class NumpyDtypeType(NumbaType, minitypes.ObjectType):
    is_numpy_dtype = True
    dtype = None # NumPy dtype type

    def resolve(self):
        return map_dtype(self.dtype)

    def __repr__(self):
        return "NumpyDtype(%s)" % self.resolve()

class ResolvedNumpyDtypeType(NumbaType, minitypes.ObjectType):
    is_numpy_dtype = True
    dtype_type = None # numba dtype type

    def resolve(self):
        return self.dtype_type

    def __repr__(self):
        return "NumpyDtype(%s)" % self.resolve()

class TBAAType(NumbaType):
    """
    Type based alias analysis type. See numba/metadata.py.
    """

    is_tbaa = True

    def __init__(self, name, root, **kwds):
        super(TBAAType, self).__init__(**kwds)
        self.name = name
        self.root = root

    @property
    def comparison_type_list(self):
        return [self.name, self.root]

    def __repr__(self):
        return "tbaa(%s)" % self.name

class EllipsisType(NumbaType, minitypes.ObjectType):
    is_ellipsis = True

    def __eq__(self, other):
        return other.is_ellipsis

    def __repr__(self):
        return "..."

class SliceType(NumbaType, minitypes.ObjectType):
    is_slice = True

    def __eq__(self, other):
        return other.is_slice

    def __repr__(self):
        return ":"

class NewAxisType(NumbaType, minitypes.ObjectType):
    is_newaxis = True

    def __eq__(self, other):
        return other.is_newaxis

    def __repr__(self):
        return "newaxis"

class GlobalType(NumbaType, minitypes.ObjectType):
    is_global = True
    name = None

    def __repr__(self):
        return "global(%s)" % self.name

class BuiltinType(NumbaType, minitypes.ObjectType):
    is_builtin = True

    def __init__(self, name, **kwds):
        super(BuiltinType, self).__init__(**kwds)
        self.name = name
        self.func = getattr(__builtin__, name)

    def __repr__(self):
        return "builtin(%s)" % self.name

class RangeType(NumbaType, minitypes.ObjectType):
    is_range = True

    def __repr__(self):
        return "range(...)"

class NoneType(NumbaType, minitypes.ObjectType):
    is_none = True

    def __repr__(self):
        return "<type(None)>"

#------------------------------------------------------------------------
# Function Types
#------------------------------------------------------------------------

class CTypesFunctionType(NumbaType, minitypes.ObjectType):
    is_ctypes_function = True

    def __init__(self, ctypes_func, restype, argtypes, **kwds):
        super(CTypesFunctionType, self).__init__(**kwds)
        self.ctypes_func = ctypes_func
        self.signature = minitypes.FunctionType(return_type=restype,
                                                args=argtypes)

    def __repr__(self):
        return "<ctypes function %s>" % (self.signature,)

class AutojitType(NumbaType, minitypes.ObjectType):
    """
    Type for autojitting functions.
    """

    is_autojit_function = True

    def __init__(self, autojit_func, **kwds):
        super(AutojitType, self).__init__(**kwds)
        self.autojit_func = autojit_func

    def __repr__(self):
        return "<autojit(%s)>" % self.autojit_func

class JitType(NumbaType, minitypes.ObjectType):
    """
    Type for autojitting functions.
    """

    is_jit_function = True

    def __init__(self, jit_func, **kwds):
        super(JitType, self).__init__(**kwds)
        self.jit_func = jit_func

    def __repr__(self):
        return "<jit(%s)>" % self.jit_func

#------------------------------------------------------------------------
# Pointer Types
#------------------------------------------------------------------------

class NULLType(NumbaType):
    """
    Null pointer type that can be compared or assigned to any other
    pointer type.
    """

    is_null = True

    def __repr__(self):
        return "<type(NULL)>"

class CTypesPointerType(NumbaType):
    def __init__(self, pointer_type, address, **kwds):
        super(CTypesPointer, self).__init__(**kwds)
        self.pointer_type = pointer_type
        self.address = address

class SizedPointerType(NumbaType, minitypes.PointerType):
    """
    A pointer with knowledge of its range.

    E.g. an array's 'shape' or 'strides' attribute.
    This also allow tuple unpacking.
    """

    size = None
    is_sized_pointer = True

    def __repr__(self):
        return "%r<%s>" % (self.base_type.pointer(), self.size)

    def __eq__(self, other):
        if other.is_sized_pointer:
            return (self.base_type == other.base_type and
                    self.size == other.size)
        return other.is_pointer and self.base_type == other.base_type

    def __hash__(self):
        return hash(self.base_type.pointer())

class CastType(NumbaType, minitypes.ObjectType):
    """
    A type instance in user code. e.g. double(value). The Name node will have
    a cast-type with dst_type 'double'.
    """

    is_cast = True

    subtypes = ['dst_type']

    def __init__(self, dst_type, **kwds):
        super(CastType, self).__init__(**kwds)
        self.dst_type = dst_type

    def __repr__(self):
        return "<cast(%s)>" % self.dst_type

#------------------------------------------------------------------------
# Aggregate Types
#------------------------------------------------------------------------

class struct(minitypes.struct):
    __doc__ = minitypes.struct.__doc__

    def ref(self):
        return ReferenceType(self)

#    def __repr__(self):
#        return "struct(...)"

#------------------------------------------------------------------------
# References
#------------------------------------------------------------------------

class ReferenceType(NumbaType):
    """
    A reference to an (primitive or Python) object. This is passed as a
    pointer and dereferences automatically.

    Currently only supported for structs.
    """

    is_reference = True

    subtypes = ['referenced_type']

    def __init__(self, referenced_type, **kwds):
        super(ReferenceType, self).__init__(**kwds)
        self.referenced_type = referenced_type

    def to_llvm(self, context):
        return self.referenced_type.pointer().to_llvm(context)

    def __repr__(self):
        return "%r.ref()" % (self.referenced_type,)

#------------------------------------------------------------------------
# END OF TYPE DEFINITIONS
#------------------------------------------------------------------------

tuple_ = TupleType(object_, size=0)
none = NoneType()
null_type = NULLType()
intp = minitypes.npy_intp

const_qualifiers = frozenset(["const"])

numpy_array = TBAAType("numpy array", root=object_)
numpy_shape = TBAAType("numpy shape", root=intp.pointer(),
                       qualifiers=const_qualifiers)
numpy_strides = TBAAType("numpy strides", root=intp.pointer(),
                         qualifiers=const_qualifiers)
numpy_ndim = TBAAType("numpy flags", root=int_.pointer())
numpy_dtype = TBAAType("numpy dtype", root=object_)
numpy_base = TBAAType("numpy base", root=object_)
numpy_flags = TBAAType("numpy flags", root=int_.pointer())

iteration_target_type = TBAAType("iteration target", root=char.pointer())
unique_tbaa_type = TBAAType("unique", root=char.pointer())

if __name__ == '__main__':
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>"""
@brief      test log(time=3s)
"""

import sys
import os
import unittest

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from src.pyquickhelper.loghelper.flog import fLOG
from src.pyquickhelper.loghelper.run_cmd import run_cmd, parse_exception_message

class TestRunCmdException(unittest.TestCase):

    def test_run_cmd_exc(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        cmd = "unexpectedcommand"
        ex = "not affected"
        try:
            out, err = run_cmd(cmd, wait=True, log_error=False, catch_exit=True, communicate=False,
                               tell_if_no_output=120, fLOG=fLOG)
            no_exception = True
            ex = None
        except Exception as e:
            no_exception = False
            out, err = parse_exception_message(e)
            ex = e
        self.assertTrue(not no_exception)
        if out is None or err is None:
            raise Exception("A\n" + str(ex))
        if len(out) > 0:
            raise Exception("B\n" + str(ex))
        if len(err) == 0:
            raise Exception("C\n" + str(ex))

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix for travid<commit_after>"""
@brief      test log(time=3s)
"""

import sys
import os
import unittest

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from src.pyquickhelper.loghelper.flog import fLOG
from src.pyquickhelper.loghelper.run_cmd import run_cmd, parse_exception_message

class TestRunCmdException(unittest.TestCase):

    def test_run_cmd_exc(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        cmd = "unexpectedcommand"
        ex = "not affected"
        try:
            out, err = run_cmd(cmd, wait=True, log_error=False, catch_exit=True, communicate=False,
                               tell_if_no_output=120, fLOG=fLOG)
            no_exception = True
            ex = None
        except Exception as e:
            no_exception = False
            out, err = parse_exception_message(e)
            ex = e
        self.assertTrue(not no_exception)
        if sys.platform.startswith("win"):
            if out is None or err is None:
                raise Exception("A\n" + str(ex))
            if len(out) > 0:
                raise Exception("B\n" + str(ex))
            if len(err) == 0:
                raise Exception("C\n" + str(ex))
        else:
            self.assertTrue(out is None)
            self.assertTrue(err is None)
            self.assertTrue(isinstance(ex, Exception))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>