<commit_before># coding: utf-8
#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################
""" Utilities to create a Jquery DataTable for your HTML file.

.. autosummary::

    DataTableFunction
    DataTable

"""
from collections import OrderedDict

import colorlog
logger = colorlog.getLogger(__name__)

class DataTableFunction(object):
    """ Class that contains Jquery DataTables function and options.

    Example:

    ::

        import pandas as pd
        from sequana.utils import DataTableFunction

        df = pandas.read_csv('data.csv')
        datatable_js = DataTableFunction(df, 'data')
        datatable_js.datatable_options = {'pageLength': 15,
                                          'dom': 'Bfrtip',
                                          'buttons': ['copy', 'csv']}
        js = datatable_js.create_javascript_function()
        html_datatables = [DataTable(df, "data_{0}".format(i), datatable_js)
                           for i, df in enumerate(df_list)]

    Here, the datatable_options dictionary is used to fine tune the appearance
    of the table.

    .. note::  DataTables add a number of elements around the table to control
        the table or show additional information about it. There are controlled
        by the order in the document (**DOM**) defined as a string made of
        letters, each of them having a precise meaning. The order of the letter
        is important. For instance if **B** is first, the buttons are put before
        the table. If **B** is at the end, it is shown below the table.
        Here are some of the valid letters and their meaning:

        - **B**: add the Buttons (copy/csv)
        - **i**: add *showing 1 to N of M entries*
        - **f**: add a search bar (**f** filtering)
        - **r**: processing display element
        - **t**: the table itself
        - **p**: pagination control

        Each option can be specified multiple times (with the exception of the
        table itself).

    .. note:: other useful options are:

        - pageLength: 15
        - scrollX: "true"
        - paging: 15
        - buttons: ['copy', 'csv']

        Note that buttons can also be excel, pdf, print, ...

    All options of datatable:
        https://datatables.net/reference/option/
    """
    def __init__(self, df, html_id, index=False):
        """.. rubric:: contructor

        :param df: data frame.
        :param str html_id: the ID used in the HTML file.
        """
        self.index = index
        self._html_id = html_id
        self._datatable_options = dict()
        self._datatable_columns = self._set_datatable_columns(df)

    @property
    def html_id(self):
        """ Get the html_id, which cannot be set by the user after the
        instanciation of the class.
        """
        return self._html_id

    @property
    def datatable_options(self):
        """ Get, set or delete the DataTable options. Setter takes a dict as
        parameter with the desired options and updates the current dictionary.

        Example::

            datatable = DataTableFunction("tab")
            datatable.datatable_options = {'dom': 'Bfrtip',
                                           'buttons': ['copy', 'csv']}

        source: https://datatables.net/reference/option/
        """
        return self._datatable_options

    @datatable_options.setter
    def datatable_options(self, d):
        try:
            d['buttons'] = self._add_export_visible(d['buttons'])
        except KeyError:
            pass
        self._datatable_options.update(d)

    def _add_export_visible(self, buttons):
        """ Add option to disable the exporting of hidden columns
        """
        try:
            for b in buttons:
                b.update({'exportOptions': {'columns': ':visible'}})
        except AttributeError:
            buttons = [{'extend': b, 'exportOptions': {'columns': ':visible'}}
                      for b in buttons]
        return buttons

    @datatable_options.deleter
    def datatable_options(self):
        self._datatable_options = dict()

    @property
    def datatable_columns(self):
        """ Get datatable_columns dictionary. It is automatically set from the
        dataframe you want to plot.
        """
        return self._datatable_columns

    def _set_datatable_columns(self, df):
        """ Fill :attr:`DataTableFunction.datatable_columns` with header of
        :param:`DataTableFunction.df`.
        """
        from pandas import Series
        if isinstance(df, Series):
            return {}

        if self.index is True:
            columns = [""] + list(df.columns)
        else:
            columns = list(df.columns)
        column_dict = OrderedDict((name, dict()) for name in columns)
        return column_dict

    def create_javascript_function(self):
        """ Return javascript to create the DataTable.
        """
        js_function = """

"""
        return js_function.format(self.html_id,
                                  self._create_datatable_option())

    def _create_datatable_option(self):
        """ Return DataTable options.
        """
        self.datatable_options['columns'] = self._create_columns_option()
        js = self._dict_to_string(self.datatable_options)
        js = "$(id).DataTable({{{0},data: results.data}});".format(js)
        return js

    def _create_columns_option(self):
        """ Return string well formated with all columns options.
        """
        js = [self._coloption_2_str(key, value) for key, value in
              self.datatable_columns.items()]
        return '[{0}]'.format(',\n'.join(js))

    def _coloption_2_str(self, name, options):
        s = "data:'{0}'".format(name)
        if options:
            s = "{0},\n{1}".format(s, self._dict_to_string(options))
        return '{{{0}}}'.format(s)

    def _dict_to_string(self, d):
        """ Convert dict to string for CanvasJS.

        Example:

        ::
            dico = {'key1': value1, 'key2': value2, 'key3': value3}
            print(CanvasJS._dict_to_string(dico))

            "key1:value1,key2:value2,key3:value3"
        """
        s = ['{0}:{1}'.format(key, self._check_type(value)) for key, value in
             d.items()]
        return ',\n'.join(s)

    def _check_type(self, value):
        """ Check value type to fill javascript sections. String must be
        surrounded by quotes and not boolean or integer.

        Javascript variable must not be surrounded by quotes. Custom variables
        start with 'data_'.
        """
        try:
            if not value.startswith(('true', 'false', 'function', '{', '[')):
                return "'{0}'".format(value)
        except AttributeError:
            return value
        return value

    def set_links_to_column(self, link_col, target_col, new_page=True):
        """Hide a column with urls and connect it with a column.

        :param str link_col: column with your URLs.
        :param str target_col: column to connect.
        """
        # hide the link column
        try:
            self.datatable_columns[link_col]['visible'] = 'false'
        except KeyError:

            keys = self.datatable_columns.keys()
            logger.warning(f"KeyError: Column name '{target_col}' does not exist. Use one of {keys}")
        # function to add link
        if new_page is True:
            fct = """function(data, type, row, meta){{
                return ''+data+'';
            }}
            """.format(link_col)
        else:
            fct = """function(data, type, row, meta){{
                return ''+data+'';
            }}
            """.format(link_col)
        try:
            self.datatable_columns[target_col]['render'] = fct
        except KeyError:
            logger.warning("KeyError: Column name '{0}' does not exist."
                           .format(target_col))
            pass

    def set_tooltips_to_column(self, tooltips_col, target_col):
        """Hide a column with tooltips and connect it with a column.

        :param str tooltips_col: column with your tooltips.
        :param str target_col: column to connect.
        """
        # hide tooltips
        try:
            self.datatable_columns[tooltips_col]['visible'] = 'false'
        except KeyError:
            logger.warning("KeyError: Column name '{0}' does not exist."
                           .format(target_col))
            pass
        # function to add tooltips
        fct = """function(data, type, row, meta){{
            return ''+data+'';
        }}
        """.format(tooltips_col)
        try:
            self.datatable_columns[target_col]['render'] = fct
        except KeyError:
            logger.warning("KeyError: Column name '{0}' does not exist."
                           .format(target_col))
            pass

class DataTable(object):
    """ Class that contains html table which used a javascript function.

    You  must add in your HTML file the JS function
    (:meth:`DataTable.create_javascript_function`) and the HTML code
    (:meth:`DataTable.create_datatable`).

    Example:

    ::

        df = pandas.read_csv('data.csv')
        datatable = DataTable(df, 'data')
        datatable.datatable.datatable_options = {'pageLength': 15,
                                                 'dom': 'Bfrtip',
                                                 'buttons': ['copy', 'csv']}
        js = datatable.create_javascript_function()
        html = datatable.create_datatable()

        # Second CSV file with same format
        df2 = pandas.read_csv('data2.csv')
        datatable2 = DataTable(df2, 'data2', datatable.datatable)
        html2 = datatable.create_datatable()

    The reason to include the JS manually is that you may include many HTML
    table but need to include the JS only once.

    """
    def __init__(self, df, html_id, datatable=None, index=False):
        """.. rubric:: contructor

        :param df: data frame.
        :param str html_id: the unique ID used in the HTML file.
        :param DataTableFunction datatable: javascript function to create the
            Jquery Datatables. If None, a :class:`DataTableFunction` is
            generated from the df.
        :param bool index: indicates whether the index dataframe should
            be included in the CSV table
        """
        self.index = index
        self._df = df
        self._html_id = html_id
        if datatable:
            self.datatable = datatable
        else:
            self.datatable = DataTableFunction(df, html_id, index=index)

    def __len__(self):
        return len(self.df)

    @property
    def df(self):
        return self._df

    @property
    def html_id(self):
        return self._html_id

    def create_datatable(self, style="width:100%", **kwargs):
        """ Return string well formated to include in a HTML page.

        :param str style: CSS option of your table.
        :param dict kwargs: parameters of :meth:`pandas.DataFrame.to_csv`.
        """

        html = """

        """.format(self.html_id, self.datatable.html_id)
        html += self._create_hidden_csv(**kwargs)
        html += self._create_html_table(style)
        return html

    def _create_hidden_csv(self, **kwargs):
        """ Return the HTML code and the CSV code for your hidden CSV section.

        :param **dict kwargs: parameters of :meth:`pandas.DataFrame.to_csv`.
        """
        csv = self._df.to_csv(index=self.index, **kwargs)
        html = '
{1}
'.format(self.html_id, csv.strip())
        css = ''.format(self.html_id)
        return '{0}\n{1}\n'.format(css, html)

    def _create_html_table(self, style):
        """ Just for set some option and header.

        :param str style: css option of your table.
        """
        # set table id
        if style:
            style = 'style="{0}"'.format(style)
        html_table = (
            '
'
            .format(self.html_id, style)
        )
        # create table's header
        th = '{0}	'
        if self.index is True:
            header = [th.format("")]
            header += [th.format(name) for name in self.df]
        else:
            header = [th.format(name) for name in self.df]
        header = '

{0}

'.format("\n".join(header))
        html_table = """
    {0}
        {1}

        """.format(html_table, header)
        return html_table

    def create_javascript_function(self):
        """ Generate the javascript function to create the DataTable in a HTML
        page.
        """
        return self.datatable.create_javascript_function()
<commit_msg>Fix bug in datatable so that ID always start with 'id' and not an integer<commit_after>#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################
""" Utilities to create a Jquery DataTable for your HTML file.

.. autosummary::

    DataTableFunction
    DataTable

"""
from collections import OrderedDict

import colorlog
logger = colorlog.getLogger(__name__)

class DataTableFunction(object):
    """ Class that contains Jquery DataTables function and options.

    Example:

    ::

        import pandas as pd
        from sequana.utils import DataTableFunction

        df = pandas.read_csv('data.csv')
        datatable_js = DataTableFunction(df, 'data')
        datatable_js.datatable_options = {'pageLength': 15,
                                          'dom': 'Bfrtip',
                                          'buttons': ['copy', 'csv']}
        js = datatable_js.create_javascript_function()
        html_datatables = [DataTable(df, "data_{0}".format(i), datatable_js)
                           for i, df in enumerate(df_list)]

    Here, the datatable_options dictionary is used to fine tune the appearance
    of the table.

    .. note::  DataTables add a number of elements around the table to control
        the table or show additional information about it. There are controlled
        by the order in the document (**DOM**) defined as a string made of
        letters, each of them having a precise meaning. The order of the letter
        is important. For instance if **B** is first, the buttons are put before
        the table. If **B** is at the end, it is shown below the table.
        Here are some of the valid letters and their meaning:

        - **B**: add the Buttons (copy/csv)
        - **i**: add *showing 1 to N of M entries*
        - **f**: add a search bar (**f** filtering)
        - **r**: processing display element
        - **t**: the table itself
        - **p**: pagination control

        Each option can be specified multiple times (with the exception of the
        table itself).

    .. note:: other useful options are:

        - pageLength: 15
        - scrollX: "true"
        - paging: 15
        - buttons: ['copy', 'csv']

        Note that buttons can also be excel, pdf, print, ...

    All options of datatable:
        https://datatables.net/reference/option/
    """
    def __init__(self, df, html_id, index=False):
        """.. rubric:: contructor

        :param df: data frame.
        :param str html_id: the ID used in the HTML file.
        """
        self.index = index
        # Mars 2022. added 'id' in front of the id to avoid issue with is
        # starting with a number, which is not allowed in HTML
        self._html_id = f"id_{html_id}"
        self._datatable_options = dict()
        self._datatable_columns = self._set_datatable_columns(df)

    @property
    def html_id(self):
        """ Get the html_id, which cannot be set by the user after the
        instanciation of the class.
        """
        return self._html_id

    @property
    def datatable_options(self):
        """ Get, set or delete the DataTable options. Setter takes a dict as
        parameter with the desired options and updates the current dictionary.

        Example::

            datatable = DataTableFunction("tab")
            datatable.datatable_options = {'dom': 'Bfrtip',
                                           'buttons': ['copy', 'csv']}

        source: https://datatables.net/reference/option/
        """
        return self._datatable_options

    @datatable_options.setter
    def datatable_options(self, d):
        try:
            d['buttons'] = self._add_export_visible(d['buttons'])
        except KeyError:
            pass
        self._datatable_options.update(d)

    def _add_export_visible(self, buttons):
        """ Add option to disable the exporting of hidden columns
        """
        try:
            for b in buttons:
                b.update({'exportOptions': {'columns': ':visible'}})
        except AttributeError:
            buttons = [{'extend': b, 'exportOptions': {'columns': ':visible'}}
                      for b in buttons]
        return buttons

    @datatable_options.deleter
    def datatable_options(self):
        self._datatable_options = dict()

    @property
    def datatable_columns(self):
        """ Get datatable_columns dictionary. It is automatically set from the
        dataframe you want to plot.
        """
        return self._datatable_columns

    def _set_datatable_columns(self, df):
        """ Fill :attr:`DataTableFunction.datatable_columns` with header of
        :param:`DataTableFunction.df`.
        """
        from pandas import Series
        if isinstance(df, Series):
            return {}

        if self.index is True:
            columns = [""] + list(df.columns)
        else:
            columns = list(df.columns)
        column_dict = OrderedDict((name, dict()) for name in columns)
        return column_dict

    def create_javascript_function(self):
        """ Return javascript to create the DataTable.
        """

        js_function = """

"""
        return js_function.format(self.html_id,
                                  self._create_datatable_option())

    def _create_datatable_option(self):
        """ Return DataTable options.
        """
        self.datatable_options['columns'] = self._create_columns_option()
        js = self._dict_to_string(self.datatable_options)
        js = "$(id).DataTable({{{0},data: results.data}});".format(js)
        return js

    def _create_columns_option(self):
        """ Return string well formated with all columns options.
        """
        js = [self._coloption_2_str(key, value) for key, value in
              self.datatable_columns.items()]
        return '[{0}]'.format(',\n'.join(js))

    def _coloption_2_str(self, name, options):
        s = "data:'{0}'".format(name)
        if options:
            s = "{0},\n{1}".format(s, self._dict_to_string(options))
        return '{{{0}}}'.format(s)

    def _dict_to_string(self, d):
        """ Convert dict to string for CanvasJS.

        Example:

        ::
            dico = {'key1': value1, 'key2': value2, 'key3': value3}
            print(CanvasJS._dict_to_string(dico))

            "key1:value1,key2:value2,key3:value3"
        """
        s = ['{0}:{1}'.format(key, self._check_type(value)) for key, value in
             d.items()]
        return ',\n'.join(s)

    def _check_type(self, value):
        """ Check value type to fill javascript sections. String must be
        surrounded by quotes and not boolean or integer.

        Javascript variable must not be surrounded by quotes. Custom variables
        start with 'data_'.
        """
        try:
            if not value.startswith(('true', 'false', 'function', '{', '[')):
                return "'{0}'".format(value)
        except AttributeError:
            return value
        return value

    def set_links_to_column(self, link_col, target_col, new_page=True):
        """Hide a column with urls and connect it with a column.

        :param str link_col: column with your URLs.
        :param str target_col: column to connect.
        """
        # hide the link column
        try:
            self.datatable_columns[link_col]['visible'] = 'false'
        except KeyError:

            keys = self.datatable_columns.keys()
            logger.warning(f"KeyError: Column name '{target_col}' does not exist. Use one of {keys}")
        # function to add link
        if new_page is True:
            fct = """function(data, type, row, meta){{
                return ''+data+'';
            }}
            """.format(link_col)
        else:
            fct = """function(data, type, row, meta){{
                return ''+data+'';
            }}
            """.format(link_col)
        try:
            self.datatable_columns[target_col]['render'] = fct
        except KeyError:
            logger.warning("KeyError: Column name '{0}' does not exist."
                           .format(target_col))
            pass

    def set_tooltips_to_column(self, tooltips_col, target_col):
        """Hide a column with tooltips and connect it with a column.

        :param str tooltips_col: column with your tooltips.
        :param str target_col: column to connect.
        """
        # hide tooltips
        try:
            self.datatable_columns[tooltips_col]['visible'] = 'false'
        except KeyError:
            logger.warning("KeyError: Column name '{0}' does not exist."
                           .format(target_col))
            pass
        # function to add tooltips
        fct = """function(data, type, row, meta){{
            return ''+data+'';
        }}
        """.format(tooltips_col)
        try:
            self.datatable_columns[target_col]['render'] = fct
        except KeyError:
            logger.warning("KeyError: Column name '{0}' does not exist."
                           .format(target_col))
            pass

class DataTable(object):
    """ Class that contains html table which used a javascript function.

    You  must add in your HTML file the JS function
    (:meth:`DataTable.create_javascript_function`) and the HTML code
    (:meth:`DataTable.create_datatable`).

    Example:

    ::

        df = pandas.read_csv('data.csv')
        datatable = DataTable(df, 'data')
        datatable.datatable.datatable_options = {'pageLength': 15,
                                                 'dom': 'Bfrtip',
                                                 'buttons': ['copy', 'csv']}
        js = datatable.create_javascript_function()
        html = datatable.create_datatable()

        # Second CSV file with same format
        df2 = pandas.read_csv('data2.csv')
        datatable2 = DataTable(df2, 'data2', datatable.datatable)
        html2 = datatable.create_datatable()

    The reason to include the JS manually is that you may include many HTML
    table but need to include the JS only once.

    """
    def __init__(self, df, html_id, datatable=None, index=False):
        """.. rubric:: contructor

        :param df: data frame.
        :param str html_id: the unique ID used in the HTML file.
        :param DataTableFunction datatable: javascript function to create the
            Jquery Datatables. If None, a :class:`DataTableFunction` is
            generated from the df.
        :param bool index: indicates whether the index dataframe should
            be included in the CSV table
        """
        self.index = index
        self._df = df
        # Mars 2022. added 'id' in front of the id to avoid issue with is
        # starting with a number, which is not allowed in HTML
        self._html_id = f"id_{html_id}"
        if datatable:
            self.datatable = datatable
        else:
            self.datatable = DataTableFunction(df, html_id, index=index)

    def __len__(self):
        return len(self.df)

    @property
    def df(self):
        return self._df

    @property
    def html_id(self):
        return self._html_id

    def create_datatable(self, style="width:100%", **kwargs):
        """ Return string well formated to include in a HTML page.

        :param str style: CSS option of your table.
        :param dict kwargs: parameters of :meth:`pandas.DataFrame.to_csv`.
        """

        html = """

        """.format(self.html_id, self.datatable.html_id)
        html += self._create_hidden_csv(**kwargs)
        html += self._create_html_table(style)
        return html

    def _create_hidden_csv(self, **kwargs):
        """ Return the HTML code and the CSV code for your hidden CSV section.

        :param **dict kwargs: parameters of :meth:`pandas.DataFrame.to_csv`.
        """
        csv = self._df.to_csv(index=self.index, **kwargs)
        html = '
{1}
'.format(self.html_id, csv.strip())
        css = ''.format(self.html_id)
        return '{0}\n{1}\n'.format(css, html)

    def _create_html_table(self, style):
        """ Just for set some option and header.

        :param str style: css option of your table.
        """
        # set table id
        if style:
            style = 'style="{0}"'.format(style)
        html_table = (
            '
'
            .format(self.html_id, style)
        )
        # create table's header
        th = '{0}	'
        if self.index is True:
            header = [th.format("")]
            header += [th.format(name) for name in self.df]
        else:
            header = [th.format(name) for name in self.df]
        header = '

{0}

'.format("\n".join(header))
        html_table = """
    {0}
        {1}

        """.format(html_table, header)
        return html_table

    def create_javascript_function(self):
        """ Generate the javascript function to create the DataTable in a HTML
        page.
        """
        return self.datatable.create_javascript_function()
<|endoftext|>


<commit_before>import json
import subprocess
import sys

import info
import utils

def serf(*args):
    args = list(args)
    rpc_port = info.NODE_INFO['rpc_port']
    args[1:1] = ['-rpc-addr', '127.0.0.1:{}'.format(rpc_port)]
    cmd = ['serf'] + args + ['-format=json']
    return json.loads(subprocess.check_output(cmd))

def serf_event(name, *args):
    rpc_port = info.NODE_INFO['rpc_port']
    cmd = ['serf', 'event',
           '-rpc-addr', '127.0.0.1:{}'.format(rpc_port),
           name] + list(args)
    subprocess.check_call(cmd, stdout=sys.stderr)

def _query(name, service):
    rpc_port = info.NODE_INFO['rpc_port']
    cmd = ['serf', 'query',
           '-rpc-addr', '127.0.0.1:{}'.format(rpc_port),
           '-format=json', name,
           json.dumps({'role': service})]
    out = json.loads(subprocess.check_output(cmd))
    for node, response in out['Responses'].items():
        if response.endswith('SUCCESS'):
            yield json.loads(response[:-len('SUCCESS')])

def where(service):
    return _query('where', service)

def where_actor(service):
    return _query('where_actor', service)

def is_self(node):
    return serf('info')['agent']['name'] == node

def serf_all_hosts():
    members = serf('members')['members']
    hosts = {}
    for member in members:
        if member['status'] == 'alive':
            ip = member['tags']['service']
            role = member['tags']['role']
            timestamp = member['tags']['timestamp']
            port_mapping = utils.decode_ports(member['tags']['ports'])
            hosts.setdefault(role, []).append(
                {'ip': ip,
                 'timestamp': timestamp,
                 'ports': port_mapping})
    return hosts

def serf_recent_hosts(all_hosts):
    hosts = {}
    for role, info in all_hosts.items():
        ip = max(info, key=lambda it: it['timestamp'])['ip']
        hosts[role] = ip
    return hosts
<commit_msg>Add convenience function to call plain serf<commit_after>import json
import subprocess
import sys

import info
import utils

def serf(*args):
    args = list(args)
    rpc_port = info.NODE_INFO['rpc_port']
    args[1:1] = ['-rpc-addr', '127.0.0.1:{}'.format(rpc_port)]
    cmd = ['serf'] + args + ['-format=json']
    return json.loads(subprocess.check_output(cmd))

def serf_plain(*args):
    """Call serf with regular output"""

    args = list(args)
    rpc_port = info.NODE_INFO['rpc_port']
    args[1:1] = ['-rpc-addr', '127.0.0.1:{}'.format(rpc_port)]
    cmd = ['serf'] + args
    return subprocess.check_output(cmd)

def serf_event(name, *args):
    rpc_port = info.NODE_INFO['rpc_port']
    cmd = ['serf', 'event',
           '-rpc-addr', '127.0.0.1:{}'.format(rpc_port),
           name] + list(args)
    subprocess.check_call(cmd, stdout=sys.stderr)

def _query(name, service):
    rpc_port = info.NODE_INFO['rpc_port']
    cmd = ['serf', 'query',
           '-rpc-addr', '127.0.0.1:{}'.format(rpc_port),
           '-format=json', name,
           json.dumps({'role': service})]
    out = json.loads(subprocess.check_output(cmd))
    for node, response in out['Responses'].items():
        if response.endswith('SUCCESS'):
            yield json.loads(response[:-len('SUCCESS')])

def where(service):
    return _query('where', service)

def where_actor(service):
    return _query('where_actor', service)

def is_self(node):
    return serf('info')['agent']['name'] == node

def serf_all_hosts():
    members = serf('members')['members']
    hosts = {}
    for member in members:
        if member['status'] == 'alive':
            ip = member['tags']['service']
            role = member['tags']['role']
            timestamp = member['tags']['timestamp']
            port_mapping = utils.decode_ports(member['tags']['ports'])
            hosts.setdefault(role, []).append(
                {'ip': ip,
                 'timestamp': timestamp,
                 'ports': port_mapping})
    return hosts

def serf_recent_hosts(all_hosts):
    hosts = {}
    for role, info in all_hosts.items():
        ip = max(info, key=lambda it: it['timestamp'])['ip']
        hosts[role] = ip
    return hosts
<|endoftext|>


<commit_before>"""This modules defines geocoding endpoints using the geonames database."""

import sys
import os
import urllib
import time
import tempfile
import zipfile
import json

import numpy as np
from girder.utility.model_importer import ModelImporter

#:  Get from geospatial plugin
GEOSPATIAL_FIELD = 'geo'

#: Put this in settings
_collection_name = 'minerva'
_collection = None
_folder_name = 'geonames'
_folder = None
_user = None

#: Columns in the TSV file for geonames
_columns = [
    'geonameid',
    'name',
    'asciiname',
    'alternatenames',
    'latitude',
    'longitude',
    'feature class',
    'feature code',
    'country code',
    'cc2',
    'admin1 code',
    'admin2 code',
    'admin3 code',
    'admin4 code',
    'population',
    'elevation',
    'dem',
    'timezone',
    'modification date'
]

# Actual types, but need NaN

#: Data types for the columns to optimize import
_types = {
    'geonameid': np.uint32,
    'latitude': np.float64,
    'longitude': np.float64,
    'population': np.object_,
    'elevation': np.object_,
    'dem': np.object_,
    'name': np.str_,
    'asciiname': np.str_,
    'alternatenames': np.str_,  # ascii comma seperated list
    'feature class': np.str_,
    'feature code': np.str_,
    'country code': np.str_,
    'cc2': np.str_,  # comma seperated list of 2 char fields
    'admin1 code': np.str_,
    'admin2 code': np.str_,
    'admin3 code': np.str_,
    'admin4 code': np.str_
}

def _type_or_default(default, typeclass):
    """Return a converter with the given default value."""
    def conv(val):
        try:
            val = float(val)
            if val < np.infty:
                return typeclass(val)
        except Exception:
            return default
    return conv

def _make_tuple(v):
    """Make a tuple out of CSV."""
    v = v.strip()
    if v:
        return tuple(v.split(','))
    else:
        return ()

#: conversion methods for dealing with nullable ints
_converters = {
    'population': _type_or_default(None, np.uint64),
    'elevation': _type_or_default(None, np.int32),
    'dem': _type_or_default(None, np.int32),
    'alternatenames': _make_tuple,
    'cc2': _make_tuple
}

# Set up default paths
#: Current directory
_mypath = os.path.abspath(os.path.dirname(__file__))

#: Path to geonames zip file
_allZip = os.path.join(_mypath, 'allCountries.zip')

#: URL of allCountries.zip
_allUrl = 'http://download.geonames.org/export/dump/allCountries.zip'

_tick = None

def pretty_print(num, suffix):
    """Pretty print a number."""
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return "{:.1f}{} {}".format(num, unit, suffix)
        num /= 1024.0
    return "{:.1f}{} {}".format(num, 'Y', suffix)

def progress_report(count, total, message,
                    unit=None, speed_fmt=pretty_print, stream=sys.stderr):
    """Print a progress message when output to a console."""
    global _tick
    if not stream.isatty():
        return

    if total <= 0:
        stream.write('%s: Unknown size\n')
        return

    percent = 100.0 * float(count) / total
    percent = max(min(percent, 100.9), 0.0)

    tock = time.time()
    if _tick is None:
        speed = 0
        _tick = tock
    else:
        speed = float(count) / (tock - _tick)

    if unit:
        msg = '\r%s: %4.1f%% (%s/s)' % (
            message,
            percent,
            speed_fmt(speed, unit)
        )
    else:
        msg = '\r%s: %4.1f%%' % (message, percent)

    stream.write(msg)
    stream.flush()

def done_report(stream=sys.stderr):
    """End the progress report message."""
    global _tick
    _tick = None
    if stream.isatty():
        stream.write('\n')

def download_all_countries(dest=_allZip, url=None,
                           progress=None, done=None):
    """Download the geonames data file to the given destination."""
    if progress is None:
        progress = progress_report
    if done is None:
        done = done_report
    if url is None:
        url = _allUrl

    message = 'Downloading allCountries.zip'
    tmp = os.path.join(tempfile.gettempdir(), 'allCountries.zip')

    try:
        urllib.urlretrieve(
            url,
            tmp,
            lambda nb, bs, fs, url=url: progress(
                nb * bs, fs, message, 'b'
            )
        )
        done()
        os.rename(tmp, dest)
    finally:
        if os.path.exists(tmp):
            os.remove(tmp)
    return dest

def export_to_girder(data, folder, user):
    """Export the geonames data to a girder folder."""
    d = data
    item = ModelImporter.model('item')

    name = d['properties']['name']
    desc = ', '.join(d['properties']['alternatenames'])
    try:
        i = item.createItem(
            name=name,
            creator=user,
            folder=folder,
            description=desc
        )
    except Exception:
        sys.stderr.write('Failed to insert item "{}"\n'.format(repr(name)))
        return

    try:
        item.setMetadata(i, d['properties'])
    except Exception:
        sys.stderr.write('Failed to write metadata:\n')
        sys.stderr.write(json.dumps(
            d,
            default=repr,
            indent=4
        ) + '\n')

    try:
        i[GEOSPATIAL_FIELD] = {
            'geometry': data['geometry']
        }
        i = item.updateItem(i)
    except Exception:
        sys.stderr.write('Failed to write geospatial data:\n')
        sys.stderr.write(json.dumps(
            i[GEOSPATIAL_FIELD],
            default=repr,
            indent=4
        ) + '\n')

def read_geonames(folder=None, user=None, file_name=_allZip, chunksize=100,
                  progress=None, done=None, handler=export_to_girder):
    """Read a geonames dump and return a pandas Dataframe."""
    import pandas

    if progress is None:
        progress = progress_report

    if done is None:
        done = done_report

    try:
        z = zipfile.ZipFile(file_name)
        f = z.open('allCountries.txt')
    except Exception:  # also try to open as a text file
        f = open(file_name)

    reader = pandas.read_csv(
        f,
        sep='\t',
        error_bad_lines=False,
        names=_columns,
        encoding='utf-8',
        parse_dates=[18],
        skip_blank_lines=True,
        index_col=None,
        chunksize=chunksize,
        dtype=_types,
        engine='c',
        converters=_converters
    )

    # alldata = pandas.DataFrame()
    n = 10200000
    for i, chunk in enumerate(reader):

        i = i * chunksize

        records = chunk.to_dict(orient='records')

        export_chunk(records, folder, user, handler)

        # there are about 10.1 million rows now
        progress(
            i, max(i, n),
            u'Importing item #{}: {}'.format(i, records[0]['name']),
            'lines'
        )

    done()

def export_to_girder(data, folder, user):
    """Export the geonames data to a girder folder."""
    item = ModelImporter.model('item')
    for d in data:
        name = d['properties']['name']
        desc = ', '.join(d['properties']['alternatenames'])
        try:
            i = item.createItem(
                name=name,
                creator=user,
                folder=folder,
                description=desc
            )
        except Exception:
            sys.stderr.write('Failed to insert item "{}"\n'.format(repr(name)))
            return

        try:
            item.setMetadata(i, d['properties'])
        except Exception:
            sys.stderr.write('Failed to write metadata:\n')
            sys.stderr.write(json.dumps(
                d,
                default=repr,
                indent=4
            ) + '\n')

        try:
            i[GEOSPATIAL_FIELD] = {
                'geometry': d['geometry']
            }
            i = item.updateItem(i)
        except Exception:
            sys.stderr.write('Failed to write geospatial data:\n')
            sys.stderr.write(json.dumps(
                i[GEOSPATIAL_FIELD],
                default=repr,
                indent=4
            ) + '\n')

def export_chunk(records, folder, user,
                 handler=export_to_girder):
    """Export the geonames data to mongo."""
    def is_nan(x):
        """Return true if the value is NaN."""
        if not isinstance(x, float):
            return False
        return not (x <= 0 or x >= 0)

    features = []
    for d in records:
        # create a geojson point feature for girder's geospatial plugin
        feature = {
            'type': 'Feature',
            'geometry': {
                'type': 'Point',
            },
            'properties': d

        }

        # remove empty/invalid fields
        for k in _columns:
            val = d.get(k)
            if is_nan(val) or val is None:
                d.pop(k)
            elif k in ('population', 'dem') and val <= 0:
                d.pop(k)

        if 'longitude' not in d or 'latitude' not in d:
            print('Invalid geometry: ' + repr(d))
            continue

        feature['geometry']['coordinates'] = [
            d.pop('longitude'),
            d.pop('latitude')
        ]
        features.append(feature)

    handler(features, folder, user)

# move to settings
def get_user():
    """Return the first user in the database."""
    # ^^ i'm a terrible person ^^
    global _user
    if _user is not None:
        return _user

    user = ModelImporter.model('user')
    return user.getAdmins().next()

def minerva_collection(user=None,
                       collection_name=_collection_name,
                       public=False):
    """Return the id of the minerva collection."""
    global _collection
    if _collection is not None and user is not None:
        return _collection

    collection = ModelImporter.model('collection')
    c = collection.findOne({'name': collection_name})

    if user is None:
        user = get_user()

    if c is None:
        # create the collection
        c = collection.createCollection(
            name=collection_name,
            creator=user,
            description='Contains global items for the Minerva plugin',
            public=public
        )
    _collection = c
    return _collection

def geonames_folder(collection=None, folder_name=_folder_name, public=False):
    """Return the folder containing geonames items."""
    global _folder
    if _folder is not None:
        return _folder

    if collection is None:
        collection = minerva_collection(public=public)

    folder = ModelImporter.model('folder')
    f = folder.findOne({
        'name': folder_name,
        'parentId': collection['_id']
    })

    if f is None:
        # create the folder
        f = folder.createFolder(
            parent=collection,
            name=folder_name,
            description='Contains the geonames database',
            parentType='collection',
            public=public
        )

    _folder = f

    return f

def main():
    """Download the allCountries file and update the database."""
    data_file = _allZip
    if not os.path.exists(_allZip):
        data_file = download_all_countries()

    read_geonames(data_file)

if __name__ == '__main__':
    main()
<commit_msg>Ignore double quotes in geonames file<commit_after>"""This modules defines geocoding endpoints using the geonames database."""

import sys
import os
import urllib
import time
import tempfile
import zipfile
import json

import numpy as np
from girder.utility.model_importer import ModelImporter

#:  Get from geospatial plugin
GEOSPATIAL_FIELD = 'geo'

#: Put this in settings
_collection_name = 'minerva'
_collection = None
_folder_name = 'geonames'
_folder = None
_user = None

#: Columns in the TSV file for geonames
_columns = [
    'geonameid',
    'name',
    'asciiname',
    'alternatenames',
    'latitude',
    'longitude',
    'feature class',
    'feature code',
    'country code',
    'cc2',
    'admin1 code',
    'admin2 code',
    'admin3 code',
    'admin4 code',
    'population',
    'elevation',
    'dem',
    'timezone',
    'modification date'
]

# Actual types, but need NaN

#: Data types for the columns to optimize import
_types = {
    'geonameid': np.uint32,
    'latitude': np.float64,
    'longitude': np.float64,
    'population': np.object_,
    'elevation': np.object_,
    'dem': np.object_,
    'name': np.str_,
    'asciiname': np.str_,
    'alternatenames': np.str_,  # ascii comma seperated list
    'feature class': np.str_,
    'feature code': np.str_,
    'country code': np.str_,
    'cc2': np.str_,  # comma seperated list of 2 char fields
    'admin1 code': np.str_,
    'admin2 code': np.str_,
    'admin3 code': np.str_,
    'admin4 code': np.str_
}

def _type_or_default(default, typeclass):
    """Return a converter with the given default value."""
    def conv(val):
        try:
            val = float(val)
            if val < np.infty:
                return typeclass(val)
        except Exception:
            return default
    return conv

def _make_tuple(v):
    """Make a tuple out of CSV."""
    v = v.strip()
    if v:
        return tuple(v.split(','))
    else:
        return ()

#: conversion methods for dealing with nullable ints
_converters = {
    'population': _type_or_default(None, np.uint64),
    'elevation': _type_or_default(None, np.int32),
    'dem': _type_or_default(None, np.int32),
    'alternatenames': _make_tuple,
    'cc2': _make_tuple
}

# Set up default paths
#: Current directory
_mypath = os.path.abspath(os.path.dirname(__file__))

#: Path to geonames zip file
_allZip = os.path.join(_mypath, 'allCountries.zip')

#: URL of allCountries.zip
_allUrl = 'http://download.geonames.org/export/dump/allCountries.zip'

_tick = None

def pretty_print(num, suffix):
    """Pretty print a number."""
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return "{:.1f}{} {}".format(num, unit, suffix)
        num /= 1024.0
    return "{:.1f}{} {}".format(num, 'Y', suffix)

def progress_report(count, total, message,
                    unit=None, speed_fmt=pretty_print, stream=sys.stderr):
    """Print a progress message when output to a console."""
    global _tick
    if not stream.isatty():
        return

    if total <= 0:
        stream.write('%s: Unknown size\n')
        return

    percent = 100.0 * float(count) / total
    percent = max(min(percent, 100.9), 0.0)

    tock = time.time()
    if _tick is None:
        speed = 0
        _tick = tock
    else:
        speed = float(count) / (tock - _tick)

    if unit:
        msg = '\r%s: %4.1f%% (%s/s)' % (
            message,
            percent,
            speed_fmt(speed, unit)
        )
    else:
        msg = '\r%s: %4.1f%%' % (message, percent)

    stream.write(msg)
    stream.flush()

def done_report(stream=sys.stderr):
    """End the progress report message."""
    global _tick
    _tick = None
    if stream.isatty():
        stream.write('\n')

def download_all_countries(dest=_allZip, url=None,
                           progress=None, done=None):
    """Download the geonames data file to the given destination."""
    if progress is None:
        progress = progress_report
    if done is None:
        done = done_report
    if url is None:
        url = _allUrl

    message = 'Downloading allCountries.zip'
    tmp = os.path.join(tempfile.gettempdir(), 'allCountries.zip')

    try:
        urllib.urlretrieve(
            url,
            tmp,
            lambda nb, bs, fs, url=url: progress(
                nb * bs, fs, message, 'b'
            )
        )
        done()
        os.rename(tmp, dest)
    finally:
        if os.path.exists(tmp):
            os.remove(tmp)
    return dest

def export_to_girder(data, folder, user):
    """Export the geonames data to a girder folder."""
    d = data
    item = ModelImporter.model('item')

    name = d['properties']['name']
    desc = ', '.join(d['properties']['alternatenames'])
    try:
        i = item.createItem(
            name=name,
            creator=user,
            folder=folder,
            description=desc
        )
    except Exception:
        sys.stderr.write('Failed to insert item "{}"\n'.format(repr(name)))
        return

    try:
        item.setMetadata(i, d['properties'])
    except Exception:
        sys.stderr.write('Failed to write metadata:\n')
        sys.stderr.write(json.dumps(
            d,
            default=repr,
            indent=4
        ) + '\n')

    try:
        i[GEOSPATIAL_FIELD] = {
            'geometry': data['geometry']
        }
        i = item.updateItem(i)
    except Exception:
        sys.stderr.write('Failed to write geospatial data:\n')
        sys.stderr.write(json.dumps(
            i[GEOSPATIAL_FIELD],
            default=repr,
            indent=4
        ) + '\n')

def read_geonames(folder=None, user=None, file_name=_allZip, chunksize=100,
                  progress=None, done=None, handler=export_to_girder):
    """Read a geonames dump and return a pandas Dataframe."""
    import pandas

    if progress is None:
        progress = progress_report

    if done is None:
        done = done_report

    try:
        z = zipfile.ZipFile(file_name)
        f = z.open('allCountries.txt')
    except Exception:  # also try to open as a text file
        f = open(file_name)

    reader = pandas.read_csv(
        f,
        sep='\t',
        error_bad_lines=False,
        names=_columns,
        encoding='utf-8',
        parse_dates=[18],
        skip_blank_lines=True,
        index_col=None,
        chunksize=chunksize,
        dtype=_types,
        engine='c',
        converters=_converters,
        quoting=3
    )

    # alldata = pandas.DataFrame()
    n = 10200000
    for i, chunk in enumerate(reader):

        i = i * chunksize

        records = chunk.to_dict(orient='records')

        export_chunk(records, folder, user, handler)

        # there are about 10.1 million rows now
        progress(
            i, max(i, n),
            u'Importing item #{}: {}'.format(i, records[0]['name']),
            'lines'
        )

    done()

def export_to_girder(data, folder, user):
    """Export the geonames data to a girder folder."""
    item = ModelImporter.model('item')
    for d in data:
        name = d['properties']['name']
        desc = ', '.join(d['properties']['alternatenames'])
        try:
            i = item.createItem(
                name=name,
                creator=user,
                folder=folder,
                description=desc
            )
        except Exception:
            sys.stderr.write('Failed to insert item "{}"\n'.format(repr(name)))
            return

        try:
            item.setMetadata(i, d['properties'])
        except Exception:
            sys.stderr.write('Failed to write metadata:\n')
            sys.stderr.write(json.dumps(
                d,
                default=repr,
                indent=4
            ) + '\n')

        try:
            i[GEOSPATIAL_FIELD] = {
                'geometry': d['geometry']
            }
            i = item.updateItem(i)
        except Exception:
            sys.stderr.write('Failed to write geospatial data:\n')
            sys.stderr.write(json.dumps(
                i[GEOSPATIAL_FIELD],
                default=repr,
                indent=4
            ) + '\n')

def export_chunk(records, folder, user,
                 handler=export_to_girder):
    """Export the geonames data to mongo."""
    def is_nan(x):
        """Return true if the value is NaN."""
        if not isinstance(x, float):
            return False
        return not (x <= 0 or x >= 0)

    features = []
    for d in records:
        # create a geojson point feature for girder's geospatial plugin
        feature = {
            'type': 'Feature',
            'geometry': {
                'type': 'Point',
            },
            'properties': d

        }

        # remove empty/invalid fields
        for k in _columns:
            val = d.get(k)
            if is_nan(val) or val is None:
                d.pop(k)
            elif k in ('population', 'dem') and val <= 0:
                d.pop(k)

        if 'longitude' not in d or 'latitude' not in d:
            print('Invalid geometry: ' + repr(d))
            continue

        feature['geometry']['coordinates'] = [
            d.pop('longitude'),
            d.pop('latitude')
        ]
        features.append(feature)

    handler(features, folder, user)

# move to settings
def get_user():
    """Return the first user in the database."""
    # ^^ i'm a terrible person ^^
    global _user
    if _user is not None:
        return _user

    user = ModelImporter.model('user')
    return user.getAdmins().next()

def minerva_collection(user=None,
                       collection_name=_collection_name,
                       public=False):
    """Return the id of the minerva collection."""
    global _collection
    if _collection is not None and user is not None:
        return _collection

    collection = ModelImporter.model('collection')
    c = collection.findOne({'name': collection_name})

    if user is None:
        user = get_user()

    if c is None:
        # create the collection
        c = collection.createCollection(
            name=collection_name,
            creator=user,
            description='Contains global items for the Minerva plugin',
            public=public
        )
    _collection = c
    return _collection

def geonames_folder(collection=None, folder_name=_folder_name, public=False):
    """Return the folder containing geonames items."""
    global _folder
    if _folder is not None:
        return _folder

    if collection is None:
        collection = minerva_collection(public=public)

    folder = ModelImporter.model('folder')
    f = folder.findOne({
        'name': folder_name,
        'parentId': collection['_id']
    })

    if f is None:
        # create the folder
        f = folder.createFolder(
            parent=collection,
            name=folder_name,
            description='Contains the geonames database',
            parentType='collection',
            public=public
        )

    _folder = f

    return f

def main():
    """Download the allCountries file and update the database."""
    data_file = _allZip
    if not os.path.exists(_allZip):
        data_file = download_all_countries()

    read_geonames(data_file)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import threading
import multiprocessing
from itertools import groupby

import rethinkdb as r

from bigchaindb import Bigchain
import cryptoconditions as cc

from server.lib.models.accounts import retrieve_accounts
from server.lib.models.assets import escrow_asset, get_subcondition_indices_from_type
from init_accounts import get_bigchain

class Connector(object):

    def __init__(self, account1, account2):
        self.accounts = {}
        self.add_accounts(account1)
        self.add_accounts(account2)

    def add_accounts(self, account):
        self.accounts[account['ledger']['id']] = account

    def listen_events(self):
        listeners = []
        for ledger_id in self.accounts.keys():
            listen = threading.Thread(target=self._listen_events, args=(ledger_id,))
            listen.start()
            listeners.append(listen)

        for listen in listeners:
            listen.join()

    def handle_escrow(self, tx, ledger_id):
        print('called handle_escrow {}'.format(tx['id']))

        ilp_header = tx['transaction']['data']['payload']['ilp_header']
        destination_ledger_id = ilp_header['ledger']

        ledger = get_bigchain(ledger_id=destination_ledger_id)
        source = self.accounts[destination_ledger_id]['vk']
        to = ilp_header['account']
        asset_id = ledger.get_owned_ids(source).pop()
        sk = self.accounts[destination_ledger_id]['sk']

        condition = cc.Fulfillment.from_dict(tx['transaction']['conditions'][0]['condition']['details'])

        timelocks, _ = get_subcondition_indices_from_type(condition, cc.TimeoutFulfillment.TYPE_ID)
        expires_at = timelocks[0].expire_time.decode()

        hashlocks, _ = get_subcondition_indices_from_type(condition, cc.PreimageSha256Fulfillment.TYPE_ID)
        execution_condition = hashlocks[0].serialize_uri()

        escrow_asset(bigchain=ledger,
                     source=source,
                     to=to,
                     asset_id=asset_id,
                     sk=sk,
                     expires_at=expires_at,
                     ilp_header=ilp_header,
                     execution_condition=execution_condition)

    def handle_execute(self, tx):
        print('called handle_execute {}'.format(tx['id']))

    def _listen_events(self, ledger_id):
        ledger = get_bigchain(ledger_id=ledger_id)
        for change in r.table('bigchain').changes().run(ledger.conn):
            if change['old_val'] is None:
                self._handle_block(change['new_val'], ledger_id)

    def _handle_block(self, block, ledger_id):
        """
        1. Alice          ---> [Alice, Chloe] ledger_a
        2. Chloe          ---> [Chloe, Bob]   ledger_b
        3. [Chloe, Bob]   ---> Bob            ledger_b
        4. [Alice, Chloe] ---> Chloe          ledger_a

        1. If chloe not in current owners and if new_owners = [current_owner, chloe] ---> escrow
        2. If current_owners == [chloe] do nothing
        3. If current_owners = [chloe, new_owner] and new_owners = [bob] ---> bob fulfilled hashlock
        4. If new_owner == [chloe] do nothing
        """
        vk = self.accounts[ledger_id]['vk']

        for transaction in block['block']['transactions']:
            current_owners = transaction['transaction']['fulfillments'][0]['current_owners']
            new_owners = transaction['transaction']['conditions'][0]['new_owners']

            # 1.
            if vk not in current_owners and sorted(new_owners) == sorted([vk] + current_owners):
                print('chloe received escrow {}'.format(transaction['id']))
                self.handle_escrow(transaction, ledger_id)
            # 2.
            elif current_owners == [vk]:
                print('skip {}'.format(transaction['id']))
            # 3.
            elif vk in current_owners and vk not in new_owners:
                print('hashlock fulfilled {}'.format(transaction['id']))
                self.handle_execute(transaction)
            # 4.
            elif new_owners == [vk]:
                print('skip {}'.format(transaction['id']))

def get_connector_accounts(db='interledger'):
    b = Bigchain()
    connector_accounts = []
    accounts_db = retrieve_accounts(b, db)

    for name, accounts in groupby(sorted(accounts_db, key=lambda d: d['name']), key=lambda d: d['name']):
        accounts = list(accounts)
        if len(accounts) == 2:
            connector_accounts.append(tuple(accounts))

    return connector_accounts

def run_connector(account1, account2):
    c = Connector(account1=account1, account2=account2)
    c.listen_events()

if __name__ == '__main__':
    connector_accounts = get_connector_accounts()
    connector_procs = []

    for connector_account in connector_accounts:
        print('Starting connector: {} <--- {} ---> {}'.format(connector_account[0]['ledger']['id'],
                                                              connector_account[0]['name'],
                                                              connector_account[1]['ledger']['id']))

        connector_proc = multiprocessing.Process(target=run_connector, args=connector_account)
        connector_proc.start()
        connector_procs.append(connector_proc)

    for connector_proc in connector_procs:
        connector_proc.join()

<commit_msg>execute chain<commit_after>import threading
import multiprocessing
from itertools import groupby

import rethinkdb as r

from bigchaindb import Bigchain
import cryptoconditions as cc

from server.lib.models.accounts import retrieve_accounts
from server.lib.models.assets import escrow_asset, get_subcondition_indices_from_type, fulfill_escrow_asset
from init_accounts import get_bigchain

class Connector(object):

    def __init__(self, account1, account2):
        self.accounts = {}
        self.add_accounts(account1)
        self.add_accounts(account2)

    def add_accounts(self, account):
        self.accounts[account['ledger']['id']] = account

    def listen_events(self):
        listeners = []
        for ledger_id in self.accounts.keys():
            listen = threading.Thread(target=self._listen_events, args=(ledger_id,))
            listen.start()
            listeners.append(listen)

        for listen in listeners:
            listen.join()

    def handle_escrow(self, tx, current_ledger_id):
        print('called handle_escrow {}'.format(tx['id']))

        ilp_header = tx['transaction']['data']['payload']['ilp_header']
        if 'hops' not in ilp_header:
            ilp_header['hops'] = []
        ilp_header['hops'].append({
            'ledger': current_ledger_id,
            'txid': tx['id']
        })

        destination_ledger_id = ilp_header['ledger']

        ledger = get_bigchain(ledger_id=destination_ledger_id)
        source = self.accounts[destination_ledger_id]['vk']
        to = ilp_header['account']
        asset_id = ledger.get_owned_ids(source).pop()
        sk = self.accounts[destination_ledger_id]['sk']

        condition = cc.Fulfillment.from_dict(tx['transaction']['conditions'][0]['condition']['details'])

        timelocks, _ = get_subcondition_indices_from_type(condition, cc.TimeoutFulfillment.TYPE_ID)
        expires_at = timelocks[0].expire_time.decode()

        hashlocks, _ = get_subcondition_indices_from_type(condition, cc.PreimageSha256Fulfillment.TYPE_ID)
        execution_condition = hashlocks[0].serialize_uri()

        escrow_asset(bigchain=ledger,
                     source=source,
                     to=to,
                     asset_id=asset_id,
                     sk=sk,
                     expires_at=expires_at,
                     ilp_header=ilp_header,
                     execution_condition=execution_condition)

    def handle_execute(self, tx):
        print('called handle_execute {}'.format(tx['id']))

        ilp_header = tx['transaction']['data']['payload']['ilp_header']

        hop = ilp_header['hops'][0]

        ledger = get_bigchain(ledger_id=hop['ledger'])
        tx_escrow = ledger.get_transaction(hop['txid'])

        source = self.accounts[hop['ledger']]['vk']
        to = source
        asset_id = {
            'txid': hop['txid'],
            'cid': 0
        }
        sk = self.accounts[hop['ledger']]['sk']

        fulfillment = cc.Fulfillment.from_uri(tx['transaction']['fulfillments'][0]['fulfillment'])

        hashlocks, _ = get_subcondition_indices_from_type(fulfillment, cc.PreimageSha256Fulfillment.TYPE_ID)
        execution_fulfillment = hashlocks[0].serialize_uri()

        fulfill_escrow_asset(bigchain=ledger,
                             source=source,
                             to=to,
                             asset_id=asset_id,
                             sk=sk,
                             execution_fulfillment=execution_fulfillment)

    def _listen_events(self, ledger_id):
        ledger = get_bigchain(ledger_id=ledger_id)
        for change in r.table('bigchain').changes().run(ledger.conn):
            if change['old_val'] is None:
                self._handle_block(change['new_val'], ledger_id)

    def _handle_block(self, block, ledger_id):
        """
        1. Alice          ---> [Alice, Chloe] ledger_a
        2. Chloe          ---> [Chloe, Bob]   ledger_b
        3. [Chloe, Bob]   ---> Bob            ledger_b
        4. [Alice, Chloe] ---> Chloe          ledger_a

        1. If chloe not in current owners and if new_owners = [current_owner, chloe] ---> escrow
        2. If current_owners == [chloe] do nothing
        3. If current_owners = [chloe, new_owner] and new_owners = [bob] ---> bob fulfilled hashlock
        4. If new_owner == [chloe] do nothing
        """
        vk = self.accounts[ledger_id]['vk']

        for transaction in block['block']['transactions']:
            current_owners = transaction['transaction']['fulfillments'][0]['current_owners']
            new_owners = transaction['transaction']['conditions'][0]['new_owners']

            # 1.
            if vk not in current_owners and sorted(new_owners) == sorted([vk] + current_owners):
                print('chloe received escrow {}'.format(transaction['id']))
                self.handle_escrow(transaction, ledger_id)
            # 2.
            elif current_owners == [vk]:
                print('skip {}'.format(transaction['id']))
            # 3.
            elif vk in current_owners and vk not in new_owners:
                print('hashlock fulfilled {}'.format(transaction['id']))
                self.handle_execute(transaction)
            # 4.
            elif new_owners == [vk]:
                print('skip {}'.format(transaction['id']))

def get_connector_accounts(db='interledger'):
    b = Bigchain()
    connector_accounts = []
    accounts_db = retrieve_accounts(b, db)

    for name, accounts in groupby(sorted(accounts_db, key=lambda d: d['name']), key=lambda d: d['name']):
        accounts = list(accounts)
        if len(accounts) == 2:
            connector_accounts.append(tuple(accounts))

    return connector_accounts

def run_connector(account1, account2):
    c = Connector(account1=account1, account2=account2)
    c.listen_events()

if __name__ == '__main__':
    connector_accounts = get_connector_accounts()
    connector_procs = []

    for connector_account in connector_accounts:
        print('Starting connector: {} <--- {} ---> {}'.format(connector_account[0]['ledger']['id'],
                                                              connector_account[0]['name'],
                                                              connector_account[1]['ledger']['id']))

        connector_proc = multiprocessing.Process(target=run_connector, args=connector_account)
        connector_proc.start()
        connector_procs.append(connector_proc)

    for connector_proc in connector_procs:
        connector_proc.join()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2008/2009 Andrey Vlasovskikh
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

'''A recurisve descent parser library based on functional combinators.

Basic combinators are taken from Harrison's book ["Introduction to Functional
Programming"][1] and translated from ML into Python. See also [a Russian
translation of the book][2].

  [1]: http://www.cl.cam.ac.uk/teaching/Lectures/funprog-jrh-1996/
  [2]: http://code.google.com/p/funprog-ru/

A parser `p` is represented by a function of type:

    p :: Sequence(a), State -> (b, State)

that takes as its input a sequence of tokens of arbitrary type `a` and a
current parsing state and return a pair of a parsed token of arbitrary type
`b` and the new parsing state.

The parsing state includes the current position in the sequence being parsed and
the position of the rightmost token that has been consumed while parsing.

Parser functions are wrapped into an object of the class `Parser`. This class
implements custom operators `+` for sequential composition of parsers, `|` for
choice composition, `>>` for transforming the result of parsing. The method
`Parser.parse` provides an easier way for invoking a parser hiding details
related to a parser state:

    Parser.parse :: Parser(a, b), Sequence(a) -> b

Altough this module is able to deal with a sequences of any kind of objects, the
recommended way of using it is applying a parser to a `Sequence(Token)`.
`Token` objects are produced by a regexp-based tokenizer defined in
`funcparserlib.lexer`. By using it this way you get more readable parsing error
messages (as `Token` objects contain their position in the source file) and good
separation of lexical and syntactic levels of the grammar. See examples for more
info.

Debug messages are emitted via a `logging.Logger` object named
`"funcparserlib"`.
'''

__all__ = ['some', 'a', 'many', 'pure', 'finished', 'maybe', 'skip', 'oneplus',
    'forward_decl', 'SyntaxError']

import logging
from funcparserlib.util import SyntaxError

log = logging.getLogger('funcparserlib')
debug = False

class Parser(object):
    '''A wrapper around a parser function that defines some operators for parser
    composition.
    '''
    def __init__(self, p):
        'Wraps a parser function p into an object.'
        self.define(p)

    def named(self, name):
        'Specifies the name of the parser for more readable parsing log.'
        self.name = name
        return self

    def define(self, p):
        'Defines a parser wrapped into this object.'
        f = getattr(p, 'run', p)
        setattr(self, '_run' if debug else 'run', f)
        self.named(getattr(p, 'name', p.__doc__))
        self.always_succeeds = False

    def run(self, tokens, s):
        '''Sequence(a), State -> (b, State)

        Runs a parser wrapped into this object.
        '''
        if debug:
            log.debug('trying %s' % self.name)
        return self._run(tokens, s)

    def _run(self, tokens, s):
        raise NotImplementedError('you must define() a parser')

    def parse(self, tokens):
        '''Sequence(a) -> b

        Applies the parser to a sequence of tokens producing a parsing result.

        It provides a way to invoke a parser hiding details related to the
        parser state. Also it makes error messages more readable by specifying
        the position of the rightmost token that has been reached.
        '''
        try:
            (tree, _) = self.run(tokens, State())
            return tree
        except NoParseError, e:
            max = e.state.max
            tok = tokens[max] if max < len(tokens) else '<EOF>'
            raise ParserError(u'%s: %s' % (e.msg, tok),
                getattr(tok, 'pos', None))

    def __add__(self, other):
        '''Parser(a, b), Parser(a, c) -> Parser(a, _Tuple(b, c))

        A sequential composition of parsers.

        NOTE: The real type of the parsed value isn't always such as specified.
        Here we use dynamic typing for ignoring the tokens that are of no
        interest to the user. Also we merge parsing results into a single _Tuple
        unless the user explicitely prevents it. See also skip and >>
        combinators.
        '''
        def magic(v1, v2):
            vs = [v for v in [v1, v2] if not isinstance(v, _Ignored)]
            if len(vs) == 1:
                return vs[0]
            elif len(vs) == 2:
                if isinstance(vs[0], _Tuple):
                    return _Tuple(v1 + (v2,))
                else:
                    return _Tuple(vs)
            else:
                return _Ignored(())
        @Parser
        def _add(tokens, s):
            (v1, s2) = self.run(tokens, s)
            (v2, s3) = other.run(tokens, s2)
            return (magic(v1, v2), s3)
        # or in terms of bind and pure:
        # _add = self.bind(lambda x: other.bind(lambda y: pure(magic(x, y))))
        _add.name = '(%s , %s)' % (self.name, other.name)
        _add.always_succeeds = self.always_succeeds and other.always_succeeds
        return _add

    def __or__(self, other):
        '''Parser(a, b), Parser(a, c) -> Parser(a, b or c)

        A choice composition of two parsers.

        NOTE: Here we are not providing the exact type of the result. In a
        statically typed langage something like Either b c could be used. See
        also + combinator.
        '''
        @Parser
        def _or(tokens, s):
            try:
                return self.run(tokens, s)
            except NoParseError, e:
                return other.run(tokens, State(s.pos, e.state.max))
        _or.name = '(%s | %s)' % (self.name, other.name)
        _or.always_succeeds = self.always_succeeds or other.always_succeeds
        return _or

    def __rshift__(self, f):
        '''Parser(a, b), (b -> c) -> Parser(a, c)

        Given a function from b to c, transforms a parser of b into a parser of
        c. It is useful for transorming a parser value into another value for
        making it a part of a parse tree or an AST.

        This combinator may be thought of as a functor from b -> c to Parser(a,
        b) -> Parser(a, c).
        '''
        @Parser
        def _shift(tokens, s):
            (v, s2) = self.run(tokens, s)
            return (f(v), s2)
        # or in terms of bind and pure:
        # _shift = self.bind(lambda x: pure(f(x)))
        _shift.name = self.name
        _shift.always_succeeds = self.always_succeeds
        return _shift

    def bind(self, f):
        '''Parser(a, b), (b -> Parser(a, c)) -> Parser(a, c)

        NOTE: A monadic bind function. It is used internally to implement other
        combinators. Functions bind and pure make the Parser a Monad.
        '''
        @Parser
        def _bind(tokens, s):
            (v, s2) = self.run(tokens, s)
            return f(v).run(tokens, s2)
        _bind.name = '(%s >>=)' % self.name
        _bind.always_succeeds = self.always_succeeds
        return _bind

class State(object):
    '''A parsing state that is maintained basically for error reporting.

    It consists of the current position pos in the sequence being parsed and
    the position max of the rightmost token that has been consumed while
    parsing.
    '''
    def __init__(self, pos=0, max=0):
        self.pos = pos
        self.max = max

    def __unicode__(self):
        return unicode((self.pos, self.max))

    def __repr__(self):
        return 'State(%r, %r)' % (self.pos, self.max)

class ParserError(SyntaxError):
    'User-visible parsing error.'
    pass

class GrammarError(Exception):
    'Raised when the grammar definition itself contains errors.'
    pass

class NoParseError(Exception):
    def __init__(self, msg='', state=None):
        self.msg = msg
        self.state = state

    def __unicode__(self):
        return self.msg

    def __str__(self):
        return self.msg.encode()

class _Tuple(tuple): pass

class _Ignored(object):
    def __init__(self, value):
        self.value = value

    def __repr__(self):
        return '_Ignored(%s)' % repr(self.value)

@Parser
def finished(tokens, s):
    '''Parser(a, None)

    Throws an exception if any tokens are left in the input unparsed.
    '''
    if s.pos >= len(tokens):
        return (None, s)
    else:
        raise NoParseError('should have reached <EOF>', s)
finished.name = 'finished'

def many(p):
    '''Parser(a, b) -> Parser(a, [b])

    Returns a parser that infinitely applies the parser p to the input sequence
    of tokens while it successfully parsers them. The resulting parser returns a
    list of parsed values.
    '''
    @Parser
    def _many(tokens, s):
        'Iterative implementation preventing the stack overflow.'
        res = []
        try:
            while True:
                (v, s) = p.run(tokens, s)
                res.append(v)
        except NoParseError, e:
            return (res, e.state)
    _many.name = '{ %s }' % p.name
    _many.always_succeeds = True
    if p.always_succeeds:
        raise GrammarError('parser %s does not halt, please fix your grammar. '
            'see FAQ for details' % _many.name)
    return _many

def some(pred):
    '''(a -> bool) -> Parser(a, a)

    Returns a parser that parses a token if it satisfies a predicate pred.
    '''
    @Parser
    def _some(tokens, s):
        if s.pos >= len(tokens):
            raise NoParseError('no tokens left in the stream', s)
        else:
            t = tokens[s.pos]
            if pred(t):
                pos = s.pos + 1
                s2 = State(pos, max(pos, s.max))
                if debug:
                    log.debug(u'*matched* "%s", new state = %s' % (t, s2))
                return (t, s2)
            else:
                if debug:
                    log.debug(u'failed "%s", state = %s' % (t, s))
                raise NoParseError('got unexpected token', s)
    _some.name = '(some)'
    return _some

def a(value):
    '''Eq(a) -> Parser(a, a)

    Returns a parser that parses a token that is equal to the value value.
    '''
    name = getattr(value, 'name', value)
    return some(lambda t: t == value).named('(a "%s")' % name)

def pure(x):
    @Parser
    def _pure(_, s):
        return (x, s)
    _pure.name = '(pure %r)' % repr(x)
    _pure.always_succeeds = True
    return _pure

def maybe(p):
    '''Parser(a, b) -> Parser(a, b or None)

    Returns a parser that retuns None if parsing fails.

    NOTE: In a statically typed language, the type Maybe b could be more
    approprieate.
    '''
    return (p | pure(None)).named('[ %s ]' % p.name)

def skip(p):
    '''Parser(a, b) -> Parser(a, _Ignored(b))

    Returns a parser which results are ignored by the combinator +. It is useful
    for throwing away elements of concrete syntax (e. g. ",", ";").
    '''
    return p >> _Ignored

def oneplus(p):
    '''Parser(a, b) -> Parser(a, [b])

    Returns a parser that applies the parser p one or more times.
    '''
    q = p + many(p) >> (lambda x: [x[0]] + x[1])
    return q.named('(%s , { %s })' % (p.name, p.name))

def with_forward_decls(suspension):
    '''(None -> Parser(a, b)) -> Parser(a, b)

    Returns a parser that computes itself lazily as a result of the suspension
    provided. It is needed when some parsers contain forward references to
    parsers defined later and such references are cyclic. See examples for more
    details.
    '''
    @Parser
    def f(tokens, s):
        return suspension().run(tokens, s)
    return f

def forward_decl():
    '''None -> Parser(?, ?)

    Returns an undefined parser that can be used as a forward declaration. You
    will be able to define() it when all the parsers it depends on are
    available.
    '''
    @Parser
    def f(tokens, s):
        raise NotImplementedError('you must define() a forward_decl somewhere')
    return f

if __name__ == '__main__':
    import doctest
    doctest.testmod()

<commit_msg>Objectified parser combinators<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2008/2009 Andrey Vlasovskikh
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

'''A recurisve descent parser library based on functional combinators.

Basic combinators are taken from Harrison's book ["Introduction to Functional
Programming"][1] and translated from ML into Python. See also [a Russian
translation of the book][2].

  [1]: http://www.cl.cam.ac.uk/teaching/Lectures/funprog-jrh-1996/
  [2]: http://code.google.com/p/funprog-ru/

A parser `p` is represented by a function of type:

    p :: Sequence(a), State -> (b, State)

that takes as its input a sequence of tokens of arbitrary type `a` and a
current parsing state and return a pair of a parsed token of arbitrary type
`b` and the new parsing state.

The parsing state includes the current position in the sequence being parsed and
the position of the rightmost token that has been consumed while parsing.

Parser functions are wrapped into an object of the class `Parser`. This class
implements custom operators `+` for sequential composition of parsers, `|` for
choice composition, `>>` for transforming the result of parsing. The method
`Parser.parse` provides an easier way for invoking a parser hiding details
related to a parser state:

    Parser.parse :: Parser(a, b), Sequence(a) -> b

Altough this module is able to deal with a sequences of any kind of objects, the
recommended way of using it is applying a parser to a `Sequence(Token)`.
`Token` objects are produced by a regexp-based tokenizer defined in
`funcparserlib.lexer`. By using it this way you get more readable parsing error
messages (as `Token` objects contain their position in the source file) and good
separation of lexical and syntactic levels of the grammar. See examples for more
info.

Debug messages are emitted via a `logging.Logger` object named
`"funcparserlib"`.
'''

__all__ = ['some', 'a', 'many', 'pure', 'finished', 'maybe', 'skip', 'oneplus',
    'forward_decl', 'SyntaxError']

import sys, logging
from funcparserlib.util import SyntaxError

log = logging.getLogger('funcparserlib')
debug = False

class Parser(object):
    '''A wrapper around a parser function that defines some operators for parser
    composition.
    '''
    def named(self, name):
        'Specifies the name of the parser for a more readable parsing log.'
        self.name = name
        return self

    def parse(self, tokens):
        '''Sequence(a) -> b

        Applies the parser to a sequence of tokens producing a parsing result.

        It provides a way to invoke a parser hiding details related to the
        parser state. Also it makes error messages more readable by specifying
        the position of the rightmost token that has been reached.
        '''
        sys.setrecursionlimit(10000)
        try:
            (tree, _) = self(tokens, State())
            return tree
        except NoParseError, e:
            max = e.state.max
            tok = tokens[max] if max < len(tokens) else '<EOF>'
            raise ParserError(u'%s: %s' % (e.msg, tok),
                              getattr(tok, 'pos', None))

    def __call__(self, tokens, s):
        return NotImplementedError('an abstract parser cannot be called')

    def __str__(self):
        return self.name

    def __add__(self, other):
        '''Parser(a, b), Parser(a, c) -> Parser(a, (b, c))

        A sequential composition of parsers.

        NOTE: The real type of the parsed value isn't always such as specified.
        Here we use dynamic typing for ignoring the tokens that are of no
        interest to the user. Also we merge parsing results into a single _Tuple
        unless the user explicitely prevents it. See also skip and >>
        combinators.
        '''
        return _Add(self, other)

    def __or__(self, other):
        '''Parser(a, b), Parser(a, c) -> Parser(a, b or c)

        A choice composition of two parsers.

        NOTE: Here we are not providing the exact type of the result. In a
        statically typed langage something like Either b c could be used. See
        also + combinator.
        '''
        return _Or(self, other)

    def __rshift__(self, f):
        '''Parser(a, b), (b -> c) -> Parser(a, c)

        Given a function from b to c, transforms a parser of b into a parser of
        c. It is useful for transorming a parser value into another value for
        making it a part of a parse tree or an AST.

        This combinator may be thought of as a functor from b -> c to Parser(a,
        b) -> Parser(a, c).
        '''
        return _Rshift(self, f)

    def bind(self, f):
        '''Parser(a, b), (b -> Parser(a, c)) -> Parser(a, c)

        NOTE: A monadic bind function. It is used internally to implement other
        combinators. Functions bind and pure make the Parser a Monad.
        '''
        return _Bind(self, f)

class _Bind(Parser):
    '''A monadic bind parser.'''

    def __init__(self, p, f):
        self.p = p
        self.f = f

    def __call__(self, tokens, s):
        (v, s2) = self.p(tokens, s)
        return self.f(v)(tokens, s2)

    def __str__(self):
        return '(%s >>=)' % self.p

class _Rshift(Parser):
    '''An interpreter parser.'''

    def __init__(self, p, f):
        self.p = p
        self.f = f

    def __call__(self, tokens, s):
        (v, s2) = self.p(tokens, s)
        return (self.f(v), s2)

        # Or in terms of bind and pure:
        # return self.p.bind(lambda x: pure(self.f(x)))

    def __str__(self):
        return str(self.p)

class _Add(Parser):
    '''A sequential composition of parsers.'''

    def __init__(self, p1, p2):
        self.p1 = p1
        self.p2 = p2

    def __call__(self, tokens, s):
        def magic(v1, v2):
            vs = [v for v in [v1, v2] if not isinstance(v, _Ignored)]
            if len(vs) == 1:
                return vs[0]
            elif len(vs) == 2:
                if isinstance(vs[0], _Tuple):
                    return _Tuple(v1 + (v2,))
                else:
                    return _Tuple(vs)
            else:
                return _Ignored(())
        (v1, s2) = self.p1(tokens, s)
        (v2, s3) = self.p2(tokens, s2)
        return (magic(v1, v2), s3)

        # Or in terms of bind and pure:
        # return self.p1.bind(lambda x: self.p2.bind(lambda y: pure(magic(x, y))))

    def __str__(self):
        return '(%s , %s)' % (self.p1, self.p2)

class _Or(Parser):
    '''A choice composition of two parsers.'''

    def __init__(self, p1, p2):
        self.p1 = p1
        self.p2 = p2

    def __call__(self, tokens, s):
        try:
            return self.p1(tokens, s)
        except NoParseError, e:
            return self.p2(tokens, State(s.pos, e.state.max))

    def __str__(self):
        return '(%s | %s)' % (self.p1, self.p2)

class State(object):
    '''A parsing state that is maintained basically for error reporting.

    It consists of the current position pos in the sequence being parsed and
    the position max of the rightmost token that has been consumed while
    parsing.
    '''
    def __init__(self, pos=0, max=0):
        self.pos = pos
        self.max = max

    def __unicode__(self):
        return unicode((self.pos, self.max))

    def __repr__(self):
        return 'State(%r, %r)' % (self.pos, self.max)

class ParserError(SyntaxError):
    'User-visible parsing error.'
    pass

class GrammarError(Exception):
    'Raised when the grammar definition itself contains errors.'
    pass

class NoParseError(Exception):
    def __init__(self, msg='', state=None):
        self.msg = msg
        self.state = state

    def __unicode__(self):
        return self.msg

    def __str__(self):
        return self.msg.encode()

class _Tuple(tuple): pass

class _Ignored(object):
    def __init__(self, value):
        self.value = value

    def __repr__(self):
        return '_Ignored(%r)' % (self.value,)

class _Finished(Parser):
    '''Throws an exception if any tokens are left in the input unparsed.'''

    def __call__(self, tokens, s):
        if s.pos >= len(tokens):
            return (None, s)
        else:
            raise NoParseError('should have reached <EOF>', s)

    def __str__(self):
        return 'finished'

finished = _Finished()

class _Many(Parser):
    '''Parser(a, b) -> Parser(a, [b])

    A parser that infinitely applies the parser p to the input sequence of
    tokens while it successfully parsers them. It returns a list of parsed
    values.
    '''

    def __init__(self, p):
        self.p = p

    def __call__(self, tokens, s):
        'Iterative implementation preventing the stack overflow.'
        res = []
        try:
            while True:
                (v, s) = self.p(tokens, s)
                res.append(v)
        except NoParseError, e:
            return (res, e.state)

    def __str__(self):
        return '{ %s }' % self.p

many = _Many

class _Some(Parser):
    '''A parser that parses a token if it satisfies a predicate pred.'''

    def __init__(self, pred):
        '''(a -> bool) -> Parser(a, a)'''
        self.pred = pred

    def __call__(self, tokens, s):
        if s.pos >= len(tokens):
            raise NoParseError('no tokens left in the stream', s)
        t = tokens[s.pos]
        if self.pred(t):
            pos = s.pos + 1
            s2 = State(pos, max(pos, s.max))
            if debug:
                log.debug(u'*matched* "%s", new state = %s' % (t, s2))
            return (t, s2)
        else:
            if debug:
                log.debug(u'failed "%s", state = %s' % (t, s))
            raise NoParseError('got unexpected token', s)

    def __str__(self):
        return '(some)'

some = _Some

def a(value):
    '''Eq(a) -> Parser(a, a)

    Returns a parser that parses a token that is equal to the value value.
    '''
    name = getattr(value, 'name', value)
    return some(lambda t: t == value).named('(a "%s")' % name)

class _Pure(Parser):
    def __init__(self, x):
        self.x = x

    def __call__(self, tokens, s):
        return (self.x, s)

    def __str__(self):
        return '(pure %s)' % (x,)

pure = _Pure

def maybe(p):
    '''Parser(a, b) -> Parser(a, b or None)

    Returns a parser that retuns None if parsing fails.

    NOTE: In a statically typed language, the type Maybe b could be more
    approprieate.
    '''
    return (p | pure(None)).named('[ %s ]' % p)

def skip(p):
    '''Parser(a, b) -> Parser(a, _Ignored(b))

    Returns a parser which results are ignored by the combinator +. It is useful
    for throwing away elements of concrete syntax (e. g. ",", ";").
    '''
    return p >> _Ignored

def oneplus(p):
    '''Parser(a, b) -> Parser(a, [b])

    Returns a parser that applies the parser p one or more times.
    '''
    q = p + many(p) >> (lambda x: [x[0]] + x[1])
    return q.named('(%s , { %s })' % (p, p))

class _ForwardDecl(Parser):
    '''
    An undefined parser that can be used as a forward declaration. You will be
    able to define() it when all the parsers it depends on are available.
    '''

    def __init__(self):
        self.p = None

    def define(self, p):
        self.p = p

    def __call__(self, tokens, s):
        if self.p:
            return self.p(tokens, s)
        else:
            raise NotImplementedError('you must define() a forward_decl')

    def __str__(self):
        return str(self.p)

forward_decl = _ForwardDecl

if __name__ == '__main__':
    import doctest
    doctest.testmod()

<|endoftext|>


<commit_before>import threading

import lib.datalayer
import games.stats

import hlib.api
import hlib.database
import hlib.log
import hlib.pageable

# pylint: disable-msg=F0401
import hruntime

class UserStats(object):
  def __init__(self, user):
    # pylint: disable-msg=E1002
    super(UserStats, self).__init__()

    self.user				= user
    self.points				= 0
    self.finished_points		= 0
    self.games				= 0
    self.finished			= 0
    self.wons				= 0
    self.points_per_game		= 0.0
    self.forhont			= 0

  def __str__(self):
    return '%s: %i, %i, %f' % (self.user.name.encode('ascii', 'replace'), self.points, self.games, self.points_per_game)

  def to_api(self):
    return {
      'user':                   hlib.api.User(self.user),
      'points':                 self.finished_points,
      'games':                  self.games,
      'finished':               self.finished,
      'wons':                   self.wons,
      'ppg':                    self.points_per_game,
      'forhont':		self.forhont
    }

class PlayerStatsWrapper(games.stats.PlayerStatsWrapper):
  def default(self, key):
    return UserStats(hruntime.dbroot.users[key])

class Stats(games.stats.Stats):
  def get_records(self, start, length):
    records = self.records

    return (records[start:max(start + length, len(records) - 1)], len(records))

  def refresh_stats(self):
    new_stats = PlayerStatsWrapper()

    def __process_game(g):
      for p in g.players.values():
        if p.user.name not in new_stats:
          s = UserStats(p.user)
          new_stats[p.user.name] = s

        else:
          s = new_stats[p.user.name]

        s.points += p.points
        s.games += 1

        if g.is_finished:
          s.finished += 1
          s.finished_points += p.points

          if g.winner_player == p:
            s.wons += 1

        elif g.is_canceled or g.is_suspended:
          pass

        else:
          if p.id == g.forhont:
            s.forhont += 1

    for g in hruntime.dbroot.games.values():
      if hruntime.time - int(hruntime.app.config['stats.games.window']) > g.last_pass:
        continue

      if g.is_canceled:
        continue

      __process_game(g)

    for g in hruntime.dbroot.games_archived.values():
      if hruntime.time - int(hruntime.app.config['stats.games.window']) > g.last_pass:
        continue

      if g.is_canceled:
        continue

      __process_game(g)

    keys_to_remove = []

    for username, stats in new_stats.items():
      if stats.finished < 20:
        keys_to_remove.append(username)

      if stats.finished > 0:
        stats.points_per_game = float(stats.finished_points) / float(stats.finished)

    for username in keys_to_remove:
      del new_stats[username]

    with self.lock:
      self._player_stats = new_stats
      self._records = sorted(new_stats.values(), key = lambda x: x.points_per_game, reverse = True)

      super(Stats, self).refresh_stats()

stats = Stats()
<commit_msg>Skip corrupted games (there are some), support last_access setting API - unused by stats<commit_after>import threading

import lib.datalayer
import games.stats

import hlib.api
import hlib.database
import hlib.log
import hlib.pageable

# pylint: disable-msg=F0401
import hruntime

class UserStats(object):
  def __init__(self, user):
    # pylint: disable-msg=E1002
    super(UserStats, self).__init__()

    self.user				= user
    self.points				= 0
    self.finished_points		= 0
    self.games				= 0
    self.finished			= 0
    self.wons				= 0
    self.points_per_game		= 0.0
    self.forhont			= 0

  def __str__(self):
    return '%s: %i, %i, %f' % (self.user.name.encode('ascii', 'replace'), self.points, self.games, self.points_per_game)

  def to_api(self):
    return {
      'user':                   hlib.api.User(self.user),
      'points':                 self.finished_points,
      'games':                  self.games,
      'finished':               self.finished,
      'wons':                   self.wons,
      'ppg':                    self.points_per_game,
      'forhont':		self.forhont
    }

class PlayerStatsWrapper(games.stats.PlayerStatsWrapper):
  def default(self, key):
    return UserStats(hruntime.dbroot.users[key])

class Stats(games.stats.Stats):
  def get_records(self, start, length):
    records = self.records

    return (records[start:max(start + length, len(records) - 1)], len(records), None)

  def refresh_stats(self):
    new_stats = PlayerStatsWrapper()

    def __process_game(g):
      for p in g.players.values():
        try:
          if p.user.name not in new_stats:
            s = UserStats(p.user)
            new_stats[p.user.name] = s

          else:
            s = new_stats[p.user.name]
        except AttributeError:
          continue

        s.points += p.points
        s.games += 1

        if g.is_finished:
          s.finished += 1
          s.finished_points += p.points

          if g.winner_player == p:
            s.wons += 1

        elif g.is_canceled or g.is_suspended:
          pass

        else:
          if p.id == g.forhont:
            s.forhont += 1

    for g in hruntime.dbroot.games.values():
      if hruntime.time - int(hruntime.app.config['stats.games.window']) > g.last_pass:
        continue

      if g.is_canceled:
        continue

      __process_game(g)

    for g in hruntime.dbroot.games_archived.values():
      if hruntime.time - int(hruntime.app.config['stats.games.window']) > g.last_pass:
        continue

      if g.is_canceled:
        continue

      __process_game(g)

    keys_to_remove = []

    for username, stats in new_stats.items():
      if stats.finished < 20:
        keys_to_remove.append(username)

      if stats.finished > 0:
        stats.points_per_game = float(stats.finished_points) / float(stats.finished)

    for username in keys_to_remove:
      del new_stats[username]

    with self.lock:
      self._player_stats = new_stats
      self._records = sorted(new_stats.values(), key = lambda x: x.points_per_game, reverse = True)

      super(Stats, self).refresh_stats()

stats = Stats()
<|endoftext|>