<commit_before>"""
homeassistant.components.binary_sensor.command_sensor
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Allows to configure custom shell commands to turn a value
into a logical value for a binary sensor.
"""
import logging
import subprocess
from datetime import timedelta

from homeassistant.const import CONF_VALUE_TEMPLATE
from homeassistant.components.binary_sensor import BinarySensorDevice
from homeassistant.util import template, Throttle

_LOGGER = logging.getLogger(__name__)

DEFAULT_NAME = "Binary Command Sensor"
DEFAULT_PAYLOAD_ON = 'ON'
DEFAULT_PAYLOAD_OFF = 'OFF'

# Return cached results if last scan was less then this time ago
MIN_TIME_BETWEEN_UPDATES = timedelta(seconds=60)

# pylint: disable=unused-argument
def setup_platform(hass, config, add_devices, discovery_info=None):
    """ Add the Command Sensor. """

    if config.get('command') is None:
        _LOGGER.error('Missing required variable: "command"')
        return False

    data = CommandSensorData(config.get('command'))

    add_devices([CommandBinarySensor(
        hass,
        data,
        config.get('name', DEFAULT_NAME),
        config.get('payload_on', DEFAULT_PAYLOAD_ON),
        config.get('payload_off', DEFAULT_PAYLOAD_OFF),
        config.get(CONF_VALUE_TEMPLATE)
    )])

# pylint: disable=too-many-arguments
class CommandBinarySensor(BinarySensorDevice):
    """ Represents a binary sensor that is returning
    a value of a shell commands. """
    def __init__(self, hass, data, name, payload_on,
                 payload_off, value_template):
        self._hass = hass
        self.data = data
        self._name = name
        self._state = False
        self._payload_on = payload_on
        self._payload_off = payload_off
        self._value_template = value_template
        self.update()

    @property
    def name(self):
        """ The name of the sensor. """
        return self._name

    @property
    def is_on(self):
        """ True if the binary sensor is on. """
        return self._state

    def update(self):
        """ Gets the latest data and updates the state. """
        self.data.update()
        value = self.data.value

        if self._value_template is not None:
            value = template.render_with_possible_json_value(
                self._hass, self.value_template, value, False)
        if value == self._payload_on:
            self._state = True
        elif value == self._payload_off:
            self._state = False

# pylint: disable=too-few-public-methods
class CommandSensorData(object):
    """ Class for handling the data retrieval. """

    def __init__(self, command):
        self.command = command
        self.value = None

    @Throttle(MIN_TIME_BETWEEN_UPDATES)
    def update(self):
        """ Gets the latest data with a shell command. """
        _LOGGER.info('Running command: %s', self.command)

        try:
            return_value = subprocess.check_output(self.command, shell=True)
            self.value = return_value.strip().decode('utf-8')
        except subprocess.CalledProcessError:
            _LOGGER.error('Command failed: %s', self.command)
<commit_msg>Update command_sensor.py<commit_after>"""
homeassistant.components.binary_sensor.command_sensor
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Allows to configure custom shell commands to turn a value
into a logical value for a binary sensor.
"""
import logging
import subprocess
from datetime import timedelta

from homeassistant.const import CONF_VALUE_TEMPLATE
from homeassistant.components.binary_sensor import BinarySensorDevice
from homeassistant.util import template, Throttle

_LOGGER = logging.getLogger(__name__)

DEFAULT_NAME = "Binary Command Sensor"
DEFAULT_PAYLOAD_ON = 'ON'
DEFAULT_PAYLOAD_OFF = 'OFF'

# Return cached results if last scan was less then this time ago
MIN_TIME_BETWEEN_UPDATES = timedelta(seconds=60)

# pylint: disable=unused-argument
def setup_platform(hass, config, add_devices, discovery_info=None):
    """ Add the Command Sensor. """

    if config.get('command') is None:
        _LOGGER.error('Missing required variable: "command"')
        return False

    data = CommandSensorData(config.get('command'))

    add_devices([CommandBinarySensor(
        hass,
        data,
        config.get('name', DEFAULT_NAME),
        config.get('payload_on', DEFAULT_PAYLOAD_ON),
        config.get('payload_off', DEFAULT_PAYLOAD_OFF),
        config.get(CONF_VALUE_TEMPLATE)
    )])

# pylint: disable=too-many-arguments
class CommandBinarySensor(BinarySensorDevice):
    """ Represents a binary sensor that is returning
    a value of a shell commands. """
    def __init__(self, hass, data, name, payload_on,
                 payload_off, value_template):
        self._hass = hass
        self.data = data
        self._name = name
        self._state = False
        self._payload_on = payload_on
        self._payload_off = payload_off
        self._value_template = value_template
        self.update()

    @property
    def name(self):
        """ The name of the sensor. """
        return self._name

    @property
    def is_on(self):
        """ True if the binary sensor is on. """
        return self._state

    def update(self):
        """ Gets the latest data and updates the state. """
        self.data.update()
        value = self.data.value

        if self._value_template is not None:
            value = template.render_with_possible_json_value(
                self._hass, self._value_template, value, False)
        if value == self._payload_on:
            self._state = True
        elif value == self._payload_off:
            self._state = False

# pylint: disable=too-few-public-methods
class CommandSensorData(object):
    """ Class for handling the data retrieval. """

    def __init__(self, command):
        self.command = command
        self.value = None

    @Throttle(MIN_TIME_BETWEEN_UPDATES)
    def update(self):
        """ Gets the latest data with a shell command. """
        _LOGGER.info('Running command: %s', self.command)

        try:
            return_value = subprocess.check_output(self.command, shell=True)
            self.value = return_value.strip().decode('utf-8')
        except subprocess.CalledProcessError:
            _LOGGER.error('Command failed: %s', self.command)
<|endoftext|>


<commit_before># Copyright 2018 OpenSynergy Indonesia
# Copyright 2020 PT. Simetri Sinergi Indonesia
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from dateutil.relativedelta import relativedelta
from openerp import _, api, fields, models
from openerp.exceptions import Warning as UserError

class HrAttendance(models.Model):
    _inherit = "hr.attendance"

    @api.multi
    @api.depends("name")
    def _compute_schedule(self):
        obj_schedule = self.env["hr.timesheet_attendance_schedule"]
        for attn in self:
            schedule = False
            company = attn.employee_id.company_id

            criteria = [
                ("employee_id", "=", attn.employee_id.id),
                ("date_start", "<=", attn.name),
                ("date_end", ">=", attn.name),
            ]
            schedules = obj_schedule.search(criteria, limit=1)
            schedule = schedules[0] if len(schedules) > 0 else False

            if not schedule:
                if attn.action == "sign_in":
                    early_buffer = fields.Datetime.from_string(
                        attn.name
                    ) + relativedelta(hours=company.early_attendance_buffer)
                    early_buffer = fields.Datetime.to_string(early_buffer)
                    criteria = [
                        ("employee_id", "=", attn.employee_id.id),
                        ("date_start", "<=", early_buffer),
                        ("date_end", ">=", attn.name),
                    ]
                    schedules = obj_schedule.search(criteria, limit=1)
                    schedule = schedules[0] if len(schedules) > 0 else False
                elif attn.action == "sign_out":
                    late_buffer = fields.Datetime.from_string(
                        attn.name
                    ) + relativedelta(hours=-company.late_attendance_buffer)
                    late_buffer = fields.Datetime.to_string(late_buffer)
                    criteria = [
                        ("employee_id", "=", attn.employee_id.id),
                        ("date_start", "<=", attn.name),
                        ("date_end", ">=", late_buffer),
                    ]
                    schedules = obj_schedule.search(criteria, limit=1)
                    schedule = schedules[0] if len(schedules) > 0 else False

            attn.schedule_id = schedule and schedule.id or False

    schedule_id = fields.Many2one(
        string="Attendance Schedule",
        comodel_name="hr.timesheet_attendance_schedule",
        compute="_compute_schedule",
        store=True,
    )

    @api.constrains(
        "schedule_id",
    )
    def _check_att_sign_in_out(self):
        for document in self:
            company = document.employee_id.company_id

            if not document.schedule_id:
                continue

            if company.max_att_sign_in > 0:
                len_att_sign_in = len(
                    document.schedule_id.attendance_ids.filtered(
                        lambda x: x.action == "sign_in"
                    )
                )
                if len_att_sign_in > company.max_att_sign_in:
                    msg = _(
                        "Total Sign In has reached maximum " "attempts per schedule"
                    )
                    raise UserError(msg)

            if company.max_att_sign_out > 0:
                len_att_sign_out = len(
                    document.schedule_id.attendance_ids.filtered(
                        lambda x: x.action == "sign_out"
                    )
                )
                if len_att_sign_out > company.max_att_sign_out:
                    msg = _(
                        "Total Sign Out has reached maximum " "attempts per schedule"
                    )
                    raise UserError(msg)
<commit_msg>[hr_timesheet_attendance_schedule] hot fix<commit_after># Copyright 2018 OpenSynergy Indonesia
# Copyright 2020 PT. Simetri Sinergi Indonesia
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from dateutil.relativedelta import relativedelta
from openerp import _, api, fields, models
from openerp.exceptions import Warning as UserError

class HrAttendance(models.Model):
    _inherit = "hr.attendance"

    @api.multi
    @api.depends("name")
    def _compute_schedule(self):
        obj_schedule = self.env["hr.timesheet_attendance_schedule"]
        for attn in self:
            schedule = False
            company = attn.employee_id.company_id

            criteria = [
                ("employee_id", "=", attn.employee_id.id),
                ("date_start", "<=", attn.name),
                ("date_end", ">=", attn.name),
            ]
            schedules = obj_schedule.search(criteria, limit=1)
            schedule = schedules[0] if len(schedules) > 0 else False

            if not schedule:
                if attn.action == "sign_in":
                    early_buffer = fields.Datetime.from_string(
                        attn.name
                    ) + relativedelta(hours=company.early_attendance_buffer)
                    early_buffer = fields.Datetime.to_string(early_buffer)
                    criteria = [
                        ("employee_id", "=", attn.employee_id.id),
                        ("date_start", "<=", early_buffer),
                        ("date_end", ">=", attn.name),
                    ]
                    schedules = obj_schedule.search(criteria)
                    schedule = schedules[-1] if len(schedules) > 0 else False
                elif attn.action == "sign_out":
                    late_buffer = fields.Datetime.from_string(
                        attn.name
                    ) + relativedelta(hours=-company.late_attendance_buffer)
                    late_buffer = fields.Datetime.to_string(late_buffer)
                    criteria = [
                        ("employee_id", "=", attn.employee_id.id),
                        ("date_start", "<=", attn.name),
                        ("date_end", ">=", late_buffer),
                    ]
                    schedules = obj_schedule.search(criteria)
                    schedule = schedules[0] if len(schedules) > 0 else False

            attn.schedule_id = schedule and schedule.id or False

    schedule_id = fields.Many2one(
        string="Attendance Schedule",
        comodel_name="hr.timesheet_attendance_schedule",
        compute="_compute_schedule",
        store=True,
    )

    @api.constrains(
        "schedule_id",
    )
    def _check_att_sign_in_out(self):
        for document in self:
            company = document.employee_id.company_id

            if not document.schedule_id:
                continue

            if company.max_att_sign_in > 0:
                len_att_sign_in = len(
                    document.schedule_id.attendance_ids.filtered(
                        lambda x: x.action == "sign_in"
                    )
                )
                if len_att_sign_in > company.max_att_sign_in:
                    msg = _(
                        "Total Sign In has reached maximum " "attempts per schedule"
                    )
                    raise UserError(msg)

            if company.max_att_sign_out > 0:
                len_att_sign_out = len(
                    document.schedule_id.attendance_ids.filtered(
                        lambda x: x.action == "sign_out"
                    )
                )
                if len_att_sign_out > company.max_att_sign_out:
                    msg = _(
                        "Total Sign Out has reached maximum " "attempts per schedule"
                    )
                    raise UserError(msg)
<|endoftext|>


<commit_before># Copyright 2015 SolidBuilds.com. All rights reserved.
#
# Authors: Ling Thio <ling.thio@gmail.com>

"""
SQLAlchemy-boolean-search
=========================
SQLAlchemy-boolean-search translates a boolean search expression such as::

    field1=*something* and not (field2==1 or field3<=10.0)

into its corresponding SQLAlchemy query filter.

Install
-------

    pip install sqlalchemy-boolean-search

Usage example
-------------

    from sqlalchemy_boolean_search import parse_boolean_search

    # DataModel defined elsewhere (with field1, field2 and field3)
    from app.models import DataModel

    # Parse boolean search into a parsed expression
    boolean_search = 'field1=*something* and not (field2==1 or field3<=10.0)'
    parsed_expression = parse_boolean_search(boolean_search)

    # Retrieve records using a filter generated by the parsed expression
    records = DataModel.query.filter(parsed_expression.filter(DataModel))

Documentation
-------------
http://sqlalchemy-boolean-search.readthedocs.org/

Authors
-------
* Ling Thio - ling.thio [at] gmail.com

Revision History
--------
2016-03-5: Modified to allow for a list of ModelClasses as input - Brian Cherinka
2016-03-11: Modified to output a dictionary of parameters: values - B. Cherinka
2016-03-16: Changed sqlalchemy values in conditions to bindparam for post-replacement - B. Cherinka
2016-03-24: Allowed for = to mean equality for non string fields and LIKE for strings - B. Cherinka
          : Changed the dot relationship in get_field to filter on the relationship_name first - B. Cherinka
"""

from __future__ import print_function
import pyparsing as pp
import inspect
from pyparsing import ParseException  # explicit export
from sqlalchemy import func, bindparam
from sqlalchemy.sql import or_, and_, not_, sqltypes

# Define a custom exception class
class BooleanSearchException(Exception):
    pass

# ***** Utility functions *****
def get_field(DataModelClass, field_name, base_name=None):
    """ Returns a SQLAlchemy Field from a field name such as 'name' or 'parent.name'.
        Returns None if no field exists by that field name.
    """
    # Handle hierarchical field names such as 'parent.name'
    if base_name:
        if base_name in DataModelClass.__tablename__:
            return getattr(DataModelClass, field_name, None)
        else:
            return None
        #relationship = getattr(DataModelClass, relationship_name)
        #return get_field(relationship.property.mapper.entity, field_name)

    # Handle flat field names such as 'name'
    return getattr(DataModelClass, field_name, None)

# ***** Define the expression element classes *****

class Condition(object):
    """ Represents a 'name operand value' condition,
        where operand can be one of: '<', '<=', '=', '==', '!=', '>=', '>'.
    """
    def __init__(self, data):
        self.fullname = data[0][0]
        if '.' in self.fullname:
            self.basename, self.name = self.fullname.split('.', 1)
        else:
            self.basename = None
            self.name = self.fullname
        self.op = data[0][1]
        self.value = data[0][2]
        if self.name not in params:
            params.update({self.fullname: self.value})

    def filter(self, DataModelClass):
        ''' Return the condition as an SQLalchemy query condition '''

        condition = None
        if inspect.ismodule(DataModelClass):
            models = [i[1] for i in inspect.getmembers(DataModelClass, inspect.isclass) if hasattr(i[1], '__tablename__')]
        else:
            if isinstance(DataModelClass, list):
                models = DataModelClass
            else:
                models = None

        if models:
            # Input is a a list of DataModelClasses
            field = None
            index = None
            for i, model in enumerate(models):

                field = get_field(model, self.name, base_name=self.basename)
                try:
                    ptype = field.type
                    ilike = field.ilike
                except AttributeError as e:
                    ptype = None
                    ilike = None

                if field and ptype and ilike:
                    index = i
                    break

            if not field:
                raise BooleanSearchException(
                    "Table '%(table_name)s' does not have a field named '%(field_name)s'."
                    % dict(table_name=model.__tablename__, field_name=self.name))

            condition = self.filter_one(models[index], field=field, condition=condition)

        else:
            # Input is only one DataModelClass
            field = get_field(DataModelClass, self.name)
            if field:
                condition = self.filter_one(DataModelClass)
            else:
                raise BooleanSearchException(
                    "Table '%(table_name)s' does not have a field named '%(field_name)s'."
                    % dict(table_name=DataModelClass.__tablename__, field_name=self.name))

        return condition

    def filter_one(self, DataModelClass, field=None, condition=None):
        """ Return the condition as a SQLAlchemy query condition
        """
        if field:
            # Prepare field and value
            lower_field = func.lower(field)
            value = self.value
            lower_value = func.lower(value)

            if field.type.python_type == float:
                try:
                    value = float(value)
                    lower_field = field
                    lower_value = value
                except:
                    raise BooleanSearchException(
                        "Field '%(name)s' expects a float value. Received value '%(value)s' instead."
                        % dict(name=self.name, value=self.value))
            elif field.type.python_type == int:
                try:
                    value = int(value)
                    lower_field = field
                    lower_value = value
                except:
                    raise BooleanSearchException(
                        "Field '%(name)s' expects an integer value. Received value '%(value)s' instead."
                        % dict(name=self.name, value=self.value))

            print('outside all ops', field, lower_field, value, lower_value, self.name, self.fullname)
            # Return SQLAlchemy condition based on operator value
            # self.name is parameter name, lower_field is Table.parameterName
            if self.op == '==':
                condition = lower_field.__eq__(bindparam(self.fullname, lower_value))
            elif self.op == '<':
                condition = lower_field.__lt__(bindparam(self.fullname, lower_value))
            elif self.op == '<=':
                condition = lower_field.__le__(bindparam(self.fullname, lower_value))
            elif self.op == '>':
                condition = lower_field.__gt__(bindparam(self.fullname, lower_value))
            elif self.op == '>=':
                condition = lower_field.__ge__(bindparam(self.fullname, lower_value))
            elif self.op == '!=':
                condition = lower_field.__ne__(bindparam(self.fullname, lower_value))
            elif self.op == '=':
                if isinstance(field.type, sqltypes.TEXT) or isinstance(field.type, sqltypes.VARCHAR):
                    # this operator maps to LIKE
                    # x=5 -> x LIKE '%5%' (x contains 5)
                    # x=5* -> x LIKE '5%' (x starts with 5)
                    field = getattr(DataModelClass, self.name)
                    value = self.value
                    print('string here', field, value, self.name, self.fullname)
                    if value.find('*') >= 0:
                        value = value.replace('*', '%')
                        condition = field.ilike(bindparam(self.fullname, value))
                    else:
                        condition = field.ilike('%'+bindparam(self.fullname, value)+'%')
                else:
                    # if not a text column, then use "=" as a straight equals
                    condition = lower_field.__eq__(bindparam(self.fullname, lower_value))

        return condition

    def __repr__(self):
        return self.fullname + self.op + self.value

class BoolNot(object):
    """ Represents the boolean operator NOT
    """
    def __init__(self, data):
        self.condition = data[0][1]
        if isinstance(self.condition, Condition) and self.condition.name not in params:
            #params.append(self.condition.name)
            params.update({self.condition.fullname: self.condition.value})

    def filter(self, DataModelClass):
        """ Return the operator as a SQLAlchemy not_() condition
        """
        return not_(self.condition.filter(DataModelClass))

    def __repr__(self):
        return 'not_(' + repr(self.condition) + ')'

class BoolAnd(object):
    """ Represents the boolean operator AND
    """
    def __init__(self, data):
        #self.conditions = [condition for condition in data[0] if condition and condition != 'and']
        self.conditions = []
        for condition in data[0]:
            if condition and condition != 'and':
                self.conditions.append(condition)
                if isinstance(condition, Condition) and condition.name not in params:
                    #params.append(condition.name)
                    params.update({condition.fullname: condition.value})

    def filter(self, DataModelClass):
        """ Return the operator as a SQLAlchemy and_() condition
        """
        conditions = [condition.filter(DataModelClass) for condition in self.conditions]
        return and_(*conditions)  # * converts list to argument sequence

    def __repr__(self):
        return 'and_(' + ', '.join([repr(condition) for condition in self.conditions]) + ')'

class BoolOr(object):
    """ Represents the boolean operator OR
    """
    def __init__(self, data):
        #self.conditions = [condition for condition in data[0] if condition and condition != 'or']
        self.conditions = []
        for condition in data[0]:
            if condition and condition != 'or':
                self.conditions.append(condition)
                if isinstance(condition, Condition) and condition.name not in params:
                    #params.append(condition.name)
                    params.update({condition.fullname: condition.value})

    def filter(self, DataModelClass):
        """ Return the operator as a SQLAlchemy or_() condition
        """
        conditions = [condition.filter(DataModelClass) for condition in self.conditions]
        return or_(*conditions)  # * converts list to argument sequence

    def __repr__(self):
        return 'or_(' + ', '.join([repr(condition) for condition in self.conditions]) + ')'

# ***** Define the boolean condition expressions *****

# Define expression elements
number = pp.Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
name = pp.Word(pp.alphas + '._', pp.alphanums + '._')
operator = pp.Regex("==|!=|<=|>=|<|>|=")
value = pp.Word(pp.alphanums + '_.*') | pp.QuotedString('"') | number
condition = pp.Group(name + operator + value)
condition.setParseAction(Condition)

# Define the expression as a hierarchy of boolean operators
# with the following precedence: NOT > AND > OR
expression_parser = pp.operatorPrecedence(condition, [
    (pp.CaselessLiteral("not"), 1, pp.opAssoc.RIGHT, BoolNot),
    (pp.CaselessLiteral("and"), 2, pp.opAssoc.LEFT, BoolAnd),
    (pp.CaselessLiteral("or"), 2, pp.opAssoc.LEFT, BoolOr),
])

params = {}

def parse_boolean_search(boolean_search):
    """ Parses the boolean search expression into a hierarchy of boolean operators.
        Returns a BoolNot or BoolAnd or BoolOr object.
    """
    global params
    params = {}
    try:
        expression = expression_parser.parseString(boolean_search)[0]
        expression.params = params
        return expression
    except ParseException as e:
        raise BooleanSearchException("Syntax error at offset %(offset)s." % dict(offset=e.col))

<commit_msg>uses Python lower instead of SQLAlchemy one<commit_after># Copyright 2015 SolidBuilds.com. All rights reserved.
#
# Authors: Ling Thio <ling.thio@gmail.com>

"""
SQLAlchemy-boolean-search
=========================
SQLAlchemy-boolean-search translates a boolean search expression such as::

    field1=*something* and not (field2==1 or field3<=10.0)

into its corresponding SQLAlchemy query filter.

Install
-------

    pip install sqlalchemy-boolean-search

Usage example
-------------

    from sqlalchemy_boolean_search import parse_boolean_search

    # DataModel defined elsewhere (with field1, field2 and field3)
    from app.models import DataModel

    # Parse boolean search into a parsed expression
    boolean_search = 'field1=*something* and not (field2==1 or field3<=10.0)'
    parsed_expression = parse_boolean_search(boolean_search)

    # Retrieve records using a filter generated by the parsed expression
    records = DataModel.query.filter(parsed_expression.filter(DataModel))

Documentation
-------------
http://sqlalchemy-boolean-search.readthedocs.org/

Authors
-------
* Ling Thio - ling.thio [at] gmail.com

Revision History
--------
2016-03-5: Modified to allow for a list of ModelClasses as input - Brian Cherinka
2016-03-11: Modified to output a dictionary of parameters: values - B. Cherinka
2016-03-16: Changed sqlalchemy values in conditions to bindparam for post-replacement - B. Cherinka
2016-03-24: Allowed for = to mean equality for non string fields and LIKE for strings - B. Cherinka
          : Changed the dot relationship in get_field to filter on the relationship_name first - B. Cherinka
"""

from __future__ import print_function
import pyparsing as pp
import inspect
from pyparsing import ParseException  # explicit export
from sqlalchemy import func, bindparam
from sqlalchemy.sql import or_, and_, not_, sqltypes

# Define a custom exception class
class BooleanSearchException(Exception):
    pass

# ***** Utility functions *****
def get_field(DataModelClass, field_name, base_name=None):
    """ Returns a SQLAlchemy Field from a field name such as 'name' or 'parent.name'.
        Returns None if no field exists by that field name.
    """
    # Handle hierarchical field names such as 'parent.name'
    if base_name:
        if base_name in DataModelClass.__tablename__:
            return getattr(DataModelClass, field_name, None)
        else:
            return None
        #relationship = getattr(DataModelClass, relationship_name)
        #return get_field(relationship.property.mapper.entity, field_name)

    # Handle flat field names such as 'name'
    return getattr(DataModelClass, field_name, None)

# ***** Define the expression element classes *****

class Condition(object):
    """ Represents a 'name operand value' condition,
        where operand can be one of: '<', '<=', '=', '==', '!=', '>=', '>'.
    """
    def __init__(self, data):
        self.fullname = data[0][0]
        if '.' in self.fullname:
            self.basename, self.name = self.fullname.split('.', 1)
        else:
            self.basename = None
            self.name = self.fullname
        self.op = data[0][1]
        self.value = data[0][2]
        if self.name not in params:
            params.update({self.fullname: self.value})

    def filter(self, DataModelClass):
        ''' Return the condition as an SQLalchemy query condition '''

        condition = None
        if inspect.ismodule(DataModelClass):
            models = [i[1] for i in inspect.getmembers(DataModelClass, inspect.isclass) if hasattr(i[1], '__tablename__')]
        else:
            if isinstance(DataModelClass, list):
                models = DataModelClass
            else:
                models = None

        if models:
            # Input is a a list of DataModelClasses
            field = None
            index = None
            for i, model in enumerate(models):

                field = get_field(model, self.name, base_name=self.basename)
                try:
                    ptype = field.type
                    ilike = field.ilike
                except AttributeError as e:
                    ptype = None
                    ilike = None

                if field and ptype and ilike:
                    index = i
                    break

            if not field:
                raise BooleanSearchException(
                    "Table '%(table_name)s' does not have a field named '%(field_name)s'."
                    % dict(table_name=model.__tablename__, field_name=self.name))

            condition = self.filter_one(models[index], field=field, condition=condition)

        else:
            # Input is only one DataModelClass
            field = get_field(DataModelClass, self.name)
            if field:
                condition = self.filter_one(DataModelClass)
            else:
                raise BooleanSearchException(
                    "Table '%(table_name)s' does not have a field named '%(field_name)s'."
                    % dict(table_name=DataModelClass.__tablename__, field_name=self.name))

        return condition

    def filter_one(self, DataModelClass, field=None, condition=None):
        """ Return the condition as a SQLAlchemy query condition
        """
        if field:
            # Prepare field and value
            lower_field = func.lower(field)
            value = self.value
            lower_value = value.lower() if hasattr(value, 'lower') else value

            if field.type.python_type == float:
                try:
                    value = float(value)
                    lower_field = field
                    lower_value = value
                except:
                    raise BooleanSearchException(
                        "Field '%(name)s' expects a float value. Received value '%(value)s' instead."
                        % dict(name=self.name, value=self.value))
            elif field.type.python_type == int:
                try:
                    value = int(value)
                    lower_field = field
                    lower_value = value
                except:
                    raise BooleanSearchException(
                        "Field '%(name)s' expects an integer value. Received value '%(value)s' instead."
                        % dict(name=self.name, value=self.value))

            # Return SQLAlchemy condition based on operator value
            # self.name is parameter name, lower_field is Table.parameterName
            if self.op == '==':
                condition = lower_field.__eq__(bindparam(self.fullname, lower_value))
            elif self.op == '<':
                condition = lower_field.__lt__(bindparam(self.fullname, lower_value))
            elif self.op == '<=':
                condition = lower_field.__le__(bindparam(self.fullname, lower_value))
            elif self.op == '>':
                condition = lower_field.__gt__(bindparam(self.fullname, lower_value))
            elif self.op == '>=':
                condition = lower_field.__ge__(bindparam(self.fullname, lower_value))
            elif self.op == '!=':
                condition = lower_field.__ne__(bindparam(self.fullname, lower_value))
            elif self.op == '=':
                if isinstance(field.type, sqltypes.TEXT) or isinstance(field.type, sqltypes.VARCHAR):
                    # this operator maps to LIKE
                    # x=5 -> x LIKE '%5%' (x contains 5)
                    # x=5* -> x LIKE '5%' (x starts with 5)
                    field = getattr(DataModelClass, self.name)
                    value = self.value
                    if value.find('*') >= 0:
                        value = value.replace('*', '%')
                        condition = field.ilike(bindparam(self.fullname, value))
                    else:
                        condition = field.ilike('%'+bindparam(self.fullname, value)+'%')
                else:
                    # if not a text column, then use "=" as a straight equals
                    condition = lower_field.__eq__(bindparam(self.fullname, lower_value))

        return condition

    def __repr__(self):
        return self.fullname + self.op + self.value

class BoolNot(object):
    """ Represents the boolean operator NOT
    """
    def __init__(self, data):
        self.condition = data[0][1]
        if isinstance(self.condition, Condition) and self.condition.name not in params:
            #params.append(self.condition.name)
            params.update({self.condition.fullname: self.condition.value})

    def filter(self, DataModelClass):
        """ Return the operator as a SQLAlchemy not_() condition
        """
        return not_(self.condition.filter(DataModelClass))

    def __repr__(self):
        return 'not_(' + repr(self.condition) + ')'

class BoolAnd(object):
    """ Represents the boolean operator AND
    """
    def __init__(self, data):
        #self.conditions = [condition for condition in data[0] if condition and condition != 'and']
        self.conditions = []
        for condition in data[0]:
            if condition and condition != 'and':
                self.conditions.append(condition)
                if isinstance(condition, Condition) and condition.name not in params:
                    #params.append(condition.name)
                    params.update({condition.fullname: condition.value})

    def filter(self, DataModelClass):
        """ Return the operator as a SQLAlchemy and_() condition
        """
        conditions = [condition.filter(DataModelClass) for condition in self.conditions]
        return and_(*conditions)  # * converts list to argument sequence

    def __repr__(self):
        return 'and_(' + ', '.join([repr(condition) for condition in self.conditions]) + ')'

class BoolOr(object):
    """ Represents the boolean operator OR
    """
    def __init__(self, data):
        #self.conditions = [condition for condition in data[0] if condition and condition != 'or']
        self.conditions = []
        for condition in data[0]:
            if condition and condition != 'or':
                self.conditions.append(condition)
                if isinstance(condition, Condition) and condition.name not in params:
                    #params.append(condition.name)
                    params.update({condition.fullname: condition.value})

    def filter(self, DataModelClass):
        """ Return the operator as a SQLAlchemy or_() condition
        """
        conditions = [condition.filter(DataModelClass) for condition in self.conditions]
        return or_(*conditions)  # * converts list to argument sequence

    def __repr__(self):
        return 'or_(' + ', '.join([repr(condition) for condition in self.conditions]) + ')'

# ***** Define the boolean condition expressions *****

# Define expression elements
number = pp.Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
name = pp.Word(pp.alphas + '._', pp.alphanums + '._')
operator = pp.Regex("==|!=|<=|>=|<|>|=")
value = pp.Word(pp.alphanums + '_.*') | pp.QuotedString('"') | number
condition = pp.Group(name + operator + value)
condition.setParseAction(Condition)

# Define the expression as a hierarchy of boolean operators
# with the following precedence: NOT > AND > OR
expression_parser = pp.operatorPrecedence(condition, [
    (pp.CaselessLiteral("not"), 1, pp.opAssoc.RIGHT, BoolNot),
    (pp.CaselessLiteral("and"), 2, pp.opAssoc.LEFT, BoolAnd),
    (pp.CaselessLiteral("or"), 2, pp.opAssoc.LEFT, BoolOr),
])

params = {}

def parse_boolean_search(boolean_search):
    """ Parses the boolean search expression into a hierarchy of boolean operators.
        Returns a BoolNot or BoolAnd or BoolOr object.
    """
    global params
    params = {}
    try:
        expression = expression_parser.parseString(boolean_search)[0]
        expression.params = params
        return expression
    except ParseException as e:
        raise BooleanSearchException("Syntax error at offset %(offset)s." % dict(offset=e.col))
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright 2012, Nachi Ueno, NTT MCL, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import logging

from quantum.common import exceptions as exc

from quantum.api.v2 import attributes
from quantum.common.utils import find_config_file
from quantum.db import api as db
from quantum.db import db_base_plugin_v2
from quantum.db import models_v2
from quantum.openstack.common import cfg
from quantum.openstack.common import importutils
from quantum.plugins.metaplugin.common import config
from quantum.plugins.metaplugin import meta_db_v2
from quantum.plugins.metaplugin.meta_models_v2 import Flavor
from quantum import policy

LOG = logging.getLogger("metaplugin")

class MetaPluginV2(db_base_plugin_v2.QuantumDbPluginV2):
    def __init__(self, configfile=None):
        LOG.debug("Start initializing metaplugin")
        options = {"sql_connection": cfg.CONF.DATABASE.sql_connection}
        options.update({'base': models_v2.model_base.BASEV2})
        sql_max_retries = cfg.CONF.DATABASE.sql_max_retries
        options.update({"sql_max_retries": sql_max_retries})
        reconnect_interval = cfg.CONF.DATABASE.reconnect_interval
        options.update({"reconnect_interval": reconnect_interval})
        self.supported_extension_aliases = \
            cfg.CONF.META.supported_extension_aliases.split(',')
        self.supported_extension_aliases.append('flavor')

        # Ignore config option overapping
        def _is_opt_registered(opts, opt):
            if opt.dest in opts:
                return True
            else:
                return False

        cfg._is_opt_registered = _is_opt_registered

        # Keep existing tables if multiple plugin use same table name.
        db.model_base.QuantumBase.__table_args__ = {'keep_existing': True}

        self.plugins = {}

        plugin_list = [plugin_set.split(':')
                       for plugin_set
                       in cfg.CONF.META.plugin_list.split(',')]
        for flavor, plugin_provider in plugin_list:
            self.plugins[flavor] = self._load_plugin(plugin_provider)

        self.extension_map = {}
        if not cfg.CONF.META.extension_map == '':
            extension_list = [method_set.split(':')
                              for method_set
                              in cfg.CONF.META.extension_map.split(',')]
            for method_name, flavor in extension_list:
                self.extension_map[method_name] = flavor

        self.default_flavor = cfg.CONF.META.default_flavor

        if not self.default_flavor in self.plugins:
            raise exc.Invalid('default_flavor %s is not plugin list' %
                              self.default_flavor)

    def _load_plugin(self, plugin_provider):
        LOG.debug("Plugin location:%s", plugin_provider)
        # If the plugin can't be found let them know gracefully
        try:
            LOG.info("Loading Plugin: %s" % plugin_provider)
            plugin_klass = importutils.import_class(plugin_provider)
        except exc.ClassNotFound:
            LOG.exception("Error loading plugin")
            raise Exception("Plugin not found.  You can install a "
                            "plugin with: pip install <plugin-name>\n"
                            "Example: pip install quantum-sample-plugin")
        return plugin_klass()

    def _get_plugin(self, flavor):
        if not flavor in self.plugins:
            raise Exception("Plugin for flavor %s not found." % flavor)
        return self.plugins[flavor]

    def __getattr__(self, key):
        # At first,  try to pickup extension command from extension_map

        if key in self.extension_map:
            flavor = self.extension_map[key]
            plugin = self._get_plugin(flavor)
            if plugin and hasattr(plugin, key):
                return getattr(plugin, key)

        # Second, try to match extension method in order of pluign list

        for flavor, plugin in self.plugins.items():
            if hasattr(plugin, key):
                return getattr(plugin, key)

        # if no plugin support the method, then raise
        raise AttributeError

    def _extend_network_dict(self, context, network):
        network['flavor:id'] = self._get_flavor_by_network_id(network['id'])

    def create_network(self, context, network):
        n = network['network']
        flavor = n.get('flavor:id')
        if not str(flavor) in self.plugins:
            flavor = self.default_flavor
        plugin = self._get_plugin(flavor)
        net = plugin.create_network(context, network)
        LOG.debug("Created network: %s with flavor %s " % (net['id'], flavor))
        try:
            meta_db_v2.add_flavor_binding(flavor, str(net['id']))
        except Exception as e:
            LOG.error('failed to add flavor bindings')
            plugin.delete_network(context, net['id'])
            raise Exception('Failed to create network')

        LOG.debug("Created network: %s" % net['id'])
        self._extend_network_dict(context, net)
        return net

    def delete_network(self, context, id):
        flavor = meta_db_v2.get_flavor_by_network(id)
        plugin = self._get_plugin(flavor)
        return plugin.delete_network(context, id)

    def get_network(self, context, id, fields=None, verbose=None):
        flavor = meta_db_v2.get_flavor_by_network(id)
        plugin = self._get_plugin(flavor)
        net = plugin.get_network(context, id, fields, verbose)
        if not fields or 'flavor:id' in fields:
            self._extend_network_dict(context, net)
        return net

    def get_networks_with_flavor(self, context, filters=None,
                                 fields=None, verbose=None):
        collection = self._model_query(context, models_v2.Network)
        collection = collection.join(Flavor,
                                     models_v2.Network.id == Flavor.network_id)
        if filters:
            for key, value in filters.iteritems():
                if key == 'flavor:id':
                    column = Flavor.flavor
                else:
                    column = getattr(models_v2.Network, key, None)
                if column:
                    collection = collection.filter(column.in_(value))
        return [self._make_network_dict(c, fields) for c in collection.all()]

    def get_networks(self, context, filters=None, fields=None, verbose=None):
        nets = self.get_networks_with_flavor(context, filters,
                                             None, verbose)
        return [self.get_network(context, net['id'],
                                 fields, verbose)
                for net in nets]

    def _get_flavor_by_network_id(self, network_id):
        return meta_db_v2.get_flavor_by_network(network_id)

    def _get_plugin_by_network_id(self, network_id):
        flavor = self._get_flavor_by_network_id(network_id)
        return self._get_plugin(flavor)

    def create_port(self, context, port):
        p = port['port']
        if not 'network_id' in p:
            raise exc.NotFound
        plugin = self._get_plugin_by_network_id(p['network_id'])
        return plugin.create_port(context, port)

    def update_port(self, context, id, port):
        port_in_db = self.get_port(context, id)
        plugin = self._get_plugin_by_network_id(port_in_db['network_id'])
        return plugin.update_port(context, id, port)

    def delete_port(self, context, id):
        port_in_db = self.get_port(context, id)
        plugin = self._get_plugin_by_network_id(port_in_db['network_id'])
        return plugin.delete_port(context, id)

    def create_subnet(self, context, subnet):
        s = subnet['subnet']
        if not 'network_id' in s:
            raise exc.NotFound
        plugin = self._get_plugin_by_network_id(s['network_id'])
        return plugin.create_subnet(context, subnet)

    def update_subnet(self, context, id, subnet):
        s = self.get_subnet(context, id)
        plugin = self._get_plugin_by_network_id(s['network_id'])
        return plugin.update_subnet(context, id, subnet)

    def delete_subnet(self, context, id):
        s = self.get_subnet(context, id)
        plugin = self._get_plugin_by_network_id(s['network_id'])
        return plugin.delete_subnet(context, id)
<commit_msg>clear db._ENGINE for each plugin init in Metaplugin<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright 2012, Nachi Ueno, NTT MCL, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import logging

from quantum.common import exceptions as exc

from quantum.api.v2 import attributes
from quantum.common.utils import find_config_file
from quantum.db import api as db
from quantum.db import db_base_plugin_v2
from quantum.db import models_v2
from quantum.openstack.common import cfg
from quantum.openstack.common import importutils
from quantum.plugins.metaplugin.common import config
from quantum.plugins.metaplugin import meta_db_v2
from quantum.plugins.metaplugin.meta_models_v2 import Flavor
from quantum import policy

LOG = logging.getLogger("metaplugin")

class MetaPluginV2(db_base_plugin_v2.QuantumDbPluginV2):
    def __init__(self, configfile=None):
        LOG.debug("Start initializing metaplugin")
        options = {"sql_connection": cfg.CONF.DATABASE.sql_connection}
        options.update({'base': models_v2.model_base.BASEV2})
        sql_max_retries = cfg.CONF.DATABASE.sql_max_retries
        options.update({"sql_max_retries": sql_max_retries})
        reconnect_interval = cfg.CONF.DATABASE.reconnect_interval
        options.update({"reconnect_interval": reconnect_interval})
        self.supported_extension_aliases = \
            cfg.CONF.META.supported_extension_aliases.split(',')
        self.supported_extension_aliases.append('flavor')

        # Ignore config option overapping
        def _is_opt_registered(opts, opt):
            if opt.dest in opts:
                return True
            else:
                return False

        cfg._is_opt_registered = _is_opt_registered

        # Keep existing tables if multiple plugin use same table name.
        db.model_base.QuantumBase.__table_args__ = {'keep_existing': True}

        self.plugins = {}

        plugin_list = [plugin_set.split(':')
                       for plugin_set
                       in cfg.CONF.META.plugin_list.split(',')]
        for flavor, plugin_provider in plugin_list:
            self.plugins[flavor] = self._load_plugin(plugin_provider)
            # Needed to clear _ENGINE for each plugin
            db._ENGINE = None

        db.configure_db(options)
        self.extension_map = {}
        if not cfg.CONF.META.extension_map == '':
            extension_list = [method_set.split(':')
                              for method_set
                              in cfg.CONF.META.extension_map.split(',')]
            for method_name, flavor in extension_list:
                self.extension_map[method_name] = flavor

        self.default_flavor = cfg.CONF.META.default_flavor

        if not self.default_flavor in self.plugins:
            raise exc.Invalid('default_flavor %s is not plugin list' %
                              self.default_flavor)

    def _load_plugin(self, plugin_provider):
        LOG.debug("Plugin location:%s", plugin_provider)
        # If the plugin can't be found let them know gracefully
        try:
            LOG.info("Loading Plugin: %s" % plugin_provider)
            plugin_klass = importutils.import_class(plugin_provider)
        except exc.ClassNotFound:
            LOG.exception("Error loading plugin")
            raise Exception("Plugin not found.  You can install a "
                            "plugin with: pip install <plugin-name>\n"
                            "Example: pip install quantum-sample-plugin")
        return plugin_klass()

    def _get_plugin(self, flavor):
        if not flavor in self.plugins:
            raise Exception("Plugin for flavor %s not found." % flavor)
        return self.plugins[flavor]

    def __getattr__(self, key):
        # At first,  try to pickup extension command from extension_map

        if key in self.extension_map:
            flavor = self.extension_map[key]
            plugin = self._get_plugin(flavor)
            if plugin and hasattr(plugin, key):
                return getattr(plugin, key)

        # Second, try to match extension method in order of pluign list

        for flavor, plugin in self.plugins.items():
            if hasattr(plugin, key):
                return getattr(plugin, key)

        # if no plugin support the method, then raise
        raise AttributeError

    def _extend_network_dict(self, context, network):
        network['flavor:id'] = self._get_flavor_by_network_id(network['id'])

    def create_network(self, context, network):
        n = network['network']
        flavor = n.get('flavor:id')
        if not str(flavor) in self.plugins:
            flavor = self.default_flavor
        plugin = self._get_plugin(flavor)
        net = plugin.create_network(context, network)
        LOG.debug("Created network: %s with flavor %s " % (net['id'], flavor))
        try:
            meta_db_v2.add_flavor_binding(flavor, str(net['id']))
        except Exception as e:
            LOG.error('failed to add flavor bindings')
            plugin.delete_network(context, net['id'])
            raise Exception('Failed to create network')

        LOG.debug("Created network: %s" % net['id'])
        self._extend_network_dict(context, net)
        return net

    def delete_network(self, context, id):
        flavor = meta_db_v2.get_flavor_by_network(id)
        plugin = self._get_plugin(flavor)
        return plugin.delete_network(context, id)

    def get_network(self, context, id, fields=None, verbose=None):
        flavor = meta_db_v2.get_flavor_by_network(id)
        plugin = self._get_plugin(flavor)
        net = plugin.get_network(context, id, fields, verbose)
        if not fields or 'flavor:id' in fields:
            self._extend_network_dict(context, net)
        return net

    def get_networks_with_flavor(self, context, filters=None,
                                 fields=None, verbose=None):
        collection = self._model_query(context, models_v2.Network)
        collection = collection.join(Flavor,
                                     models_v2.Network.id == Flavor.network_id)
        if filters:
            for key, value in filters.iteritems():
                if key == 'flavor:id':
                    column = Flavor.flavor
                else:
                    column = getattr(models_v2.Network, key, None)
                if column:
                    collection = collection.filter(column.in_(value))
        return [self._make_network_dict(c, fields) for c in collection.all()]

    def get_networks(self, context, filters=None, fields=None, verbose=None):
        nets = self.get_networks_with_flavor(context, filters,
                                             None, verbose)
        return [self.get_network(context, net['id'],
                                 fields, verbose)
                for net in nets]

    def _get_flavor_by_network_id(self, network_id):
        return meta_db_v2.get_flavor_by_network(network_id)

    def _get_plugin_by_network_id(self, network_id):
        flavor = self._get_flavor_by_network_id(network_id)
        return self._get_plugin(flavor)

    def create_port(self, context, port):
        p = port['port']
        if not 'network_id' in p:
            raise exc.NotFound
        plugin = self._get_plugin_by_network_id(p['network_id'])
        return plugin.create_port(context, port)

    def update_port(self, context, id, port):
        port_in_db = self.get_port(context, id)
        plugin = self._get_plugin_by_network_id(port_in_db['network_id'])
        return plugin.update_port(context, id, port)

    def delete_port(self, context, id):
        port_in_db = self.get_port(context, id)
        plugin = self._get_plugin_by_network_id(port_in_db['network_id'])
        return plugin.delete_port(context, id)

    def create_subnet(self, context, subnet):
        s = subnet['subnet']
        if not 'network_id' in s:
            raise exc.NotFound
        plugin = self._get_plugin_by_network_id(s['network_id'])
        return plugin.create_subnet(context, subnet)

    def update_subnet(self, context, id, subnet):
        s = self.get_subnet(context, id)
        plugin = self._get_plugin_by_network_id(s['network_id'])
        return plugin.update_subnet(context, id, subnet)

    def delete_subnet(self, context, id):
        s = self.get_subnet(context, id)
        plugin = self._get_plugin_by_network_id(s['network_id'])
        return plugin.delete_subnet(context, id)
<|endoftext|>


<commit_before>from ecnet import Server

def tune():

    sv = Server()
    sv.import_data(
        'cn_model_v1.0.csv',
        sort_type='random',
        data_split=[0.65, 0.25, 0.1]
    )
    sv.tune_hyperparameters(num_iterations=2, num_employers=2)

    sv = Server()
    sv.import_data(
        'cn_model_v1.0.csv',
        sort_type='random',
        data_split=[0.65, 0.25, 0.1]
    )
    sv.tune_hyperparameters(num_iterations=2, num_employers=2)

def tune_mp():

    sv = Server(num_processes=4)
    sv.import_data(
        'cn_model_v1.0.csv',
        sort_type='random',
        data_split=[0.65, 0.25, 0.1]
    )
    sv.tune_hyperparameters(num_iterations=2, num_employers=2)

    sv = Server()
    sv.import_data(
        'cn_model_v1.0.csv',
        sort_type='random',
        data_split=[0.65, 0.25, 0.1]
    )
    sv.tune_hyperparameters(num_iterations=2, num_employers=2)

if __name__ == '__main__':

    tune()
    tune_mp()
<commit_msg>Removed extraneous code<commit_after>from ecnet import Server

def tune():

    sv = Server()
    sv.import_data(
        'cn_model_v1.0.csv',
        sort_type='random',
        data_split=[0.65, 0.25, 0.1]
    )
    sv.tune_hyperparameters(num_iterations=2, num_employers=2)

def tune_mp():

    sv = Server(num_processes=4)
    sv.import_data(
        'cn_model_v1.0.csv',
        sort_type='random',
        data_split=[0.65, 0.25, 0.1]
    )
    sv.tune_hyperparameters(num_iterations=2, num_employers=2)

if __name__ == '__main__':

    tune()
    tune_mp()
<|endoftext|>


<commit_before>import json
import random
from unittest.mock import MagicMock

import gevent
import pytest

from raiden.constants import UINT64_MAX
from raiden.messages import Processed, SecretRequest
from raiden.network.transport import MatrixTransport
from raiden.network.transport.matrix import UserPresence, _RetryQueue
from raiden.tests.utils.factories import HOP1, HOP1_KEY, UNIT_SECRETHASH, make_address
from raiden.tests.utils.mocks import MockRaidenService
from raiden.transfer.mediated_transfer.events import CHANNEL_IDENTIFIER_GLOBAL_QUEUE
from raiden.transfer.queue_identifier import QueueIdentifier
from raiden.transfer.state_change import ActionUpdateTransportAuthData
from raiden.utils import pex
from raiden.utils.typing import Address, List, Optional, Union

USERID1 = '@Alice:Wonderland'

@pytest.fixture
def mock_matrix(
        monkeypatch,
        retry_interval,
        retries_before_backoff,
        local_matrix_server,
        private_rooms,
):

    from matrix_client.user import User
    monkeypatch.setattr(User, 'get_display_name', lambda _: 'random_display_name')

    def mock_get_user(klass, user: Union[User, str]) -> User:
        return User(None, USERID1)

    def mock_get_room_ids_for_address(
            klass,
            address: Address,
            filter_private: bool=None,
    ) -> List[str]:
        return ['!roomID:server']

    def mock_set_room_id_for_address(self, address: Address, room_id: Optional[str]):
        pass

    def mock_receive_message(klass, message):
        # We are just unit testing the matrix transport receive so do nothing
        assert message

    def mock_receive_delivered(klass, delivered):
        # We are just unit testing the matrix transport receive so do nothing
        assert delivered

    config = dict(
        retry_interval=retry_interval,
        retries_before_backoff=retries_before_backoff,
        server=local_matrix_server,
        server_name='matrix.local.raiden',
        available_servers=[],
        discovery_room='discovery',
        private_rooms=private_rooms,
    )

    transport = MatrixTransport(config)
    transport._raiden_service = MockRaidenService()
    transport._stop_event.clear()
    transport._address_to_userids[HOP1] = USERID1

    monkeypatch.setattr(MatrixTransport, '_get_user', mock_get_user)
    monkeypatch.setattr(
        MatrixTransport,
        '_get_room_ids_for_address',
        mock_get_room_ids_for_address,
    )
    monkeypatch.setattr(MatrixTransport, '_set_room_id_for_address', mock_set_room_id_for_address)
    monkeypatch.setattr(MatrixTransport, '_receive_message', mock_receive_message)

    return transport

@pytest.fixture()
def skip_userid_validation(monkeypatch):
    def mock_validate_userid_signature(klass, user):
        return HOP1

    monkeypatch.setattr(
        MatrixTransport,
        '_validate_userid_signature',
        mock_validate_userid_signature,
    )

def make_message(convert_to_hex: bool = False, overwrite_data=None):
    from matrix_client.room import Room
    room = Room(None, '!roomID:server')
    if not overwrite_data:
        message = SecretRequest(
            message_identifier=random.randint(0, UINT64_MAX),
            payment_identifier=1,
            secrethash=UNIT_SECRETHASH,
            amount=1,
            expiration=10,
        )
        message.sign(HOP1_KEY)
        data = message.encode()
        if convert_to_hex:
            data = '0x' + data.hex()
        else:
            data = json.dumps(message.to_dict())
    else:
        data = overwrite_data

    event = dict(
        type='m.room.message',
        sender=USERID1,
        content={
            'msgtype': 'm.text',
            'body': data,
        },
    )
    return room, event

def test_normal_processing_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(convert_to_hex=True)
    assert m._handle_message(room, event)

def test_normal_processing_json(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(convert_to_hex=False)
    assert m._handle_message(room, event)

def test_processing_invalid_json(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    invalid_json = '{"foo": 1,'
    room, event = make_message(convert_to_hex=False, overwrite_data=invalid_json)
    assert not m._handle_message(room, event)

def test_sending_nonstring_body(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(overwrite_data=b'somebinarydata')
    assert not m._handle_message(room, event)

def test_processing_invalid_message_json(
        mock_matrix,
        skip_userid_validation,
        skip_if_not_matrix,
):
    m = mock_matrix
    invalid_message = '{"this": 1, "message": 5, "is": 3, "not_valid": 5}'
    room, event = make_message(convert_to_hex=False, overwrite_data=invalid_message)
    assert not m._handle_message(room, event)

def test_processing_invalid_message_cmdid_json(
        mock_matrix,
        skip_userid_validation,
        skip_if_not_matrix,
):
    m = mock_matrix
    invalid_message = '{"type": "NonExistentMessage", "is": 3, "not_valid": 5}'
    room, event = make_message(convert_to_hex=False, overwrite_data=invalid_message)
    assert not m._handle_message(room, event)

def test_processing_invalid_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(convert_to_hex=True)
    old_data = event['content']['body']
    event['content']['body'] = old_data[:-1]
    assert not m._handle_message(room, event)

def test_processing_invalid_message_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(convert_to_hex=True)
    old_data = event['content']['body']
    event['content']['body'] = old_data[:-4]
    assert not m._handle_message(room, event)

def test_processing_invalid_message_cmdid_hex(
        mock_matrix,
        skip_userid_validation,
        skip_if_not_matrix,
):
    m = mock_matrix
    room, event = make_message(convert_to_hex=True)
    old_data = event['content']['body']
    event['content']['body'] = '0xff' + old_data[4:]
    assert not m._handle_message(room, event)

def test_matrix_message_sync(
        skip_if_not_matrix,
        local_matrix_server,
        private_rooms,
        retry_interval,
        retries_before_backoff,
):
    transport0 = MatrixTransport({
        'discovery_room': 'discovery',
        'retries_before_backoff': retries_before_backoff,
        'retry_interval': retry_interval,
        'server': local_matrix_server,
        'server_name': 'matrix.local.raiden',
        'available_servers': [],
        'private_rooms': private_rooms,
    })
    transport1 = MatrixTransport({
        'discovery_room': 'discovery',
        'retries_before_backoff': retries_before_backoff,
        'retry_interval': retry_interval,
        'server': local_matrix_server,
        'server_name': 'matrix.local.raiden',
        'available_servers': [],
        'private_rooms': private_rooms,
    })

    latest_auth_data = None

    received_messages = set()

    class MessageHandler:
        def on_message(self, _, message):
            nonlocal received_messages
            received_messages.add(message)

    message_handler = MessageHandler()
    raiden_service0 = MockRaidenService(message_handler)
    raiden_service1 = MockRaidenService(message_handler)

    raiden_service1.handle_state_change = MagicMock()

    transport0.start(
        raiden_service0,
        message_handler,
        None,
    )
    transport1.start(
        raiden_service1,
        message_handler,
        None,
    )

    gevent.sleep(1)

    latest_auth_data = f'{transport1._user_id}/{transport1._client.api.token}'
    update_transport_auth_data = ActionUpdateTransportAuthData(latest_auth_data)
    raiden_service1.handle_state_change.assert_called_with(update_transport_auth_data)

    transport0.start_health_check(transport1._raiden_service.address)
    transport1.start_health_check(transport0._raiden_service.address)

    queue_identifier = QueueIdentifier(
        recipient=transport1._raiden_service.address,
        channel_identifier=1,
    )

    for i in range(5):
        message = Processed(i)
        message.sign(transport0._raiden_service.private_key)
        transport0.send_async(
            queue_identifier,
            message,
        )

    gevent.sleep(2)

    assert len(received_messages) == 10
    for i in range(5):
        assert any(getattr(m, 'message_identifier', -1) == i for m in received_messages)

    transport1.stop()

    assert latest_auth_data

    # Send more messages while the other end is offline
    for i in range(10, 15):
        message = Processed(i)
        message.sign(transport0._raiden_service.private_key)
        transport0.send_async(
            queue_identifier,
            message,
        )

    # Should fetch the 5 messages sent while transport1 was offline
    transport1.start(
        transport1._raiden_service,
        message_handler,
        latest_auth_data,
    )

    gevent.sleep(2)

    assert len(set(received_messages)) == 20
    for i in range(10, 15):
        assert any(getattr(m, 'message_identifier', -1) == i for m in received_messages)

    transport0.stop()
    transport1.stop()
    transport0.get()
    transport1.get()

def test_matrix_message_retry(
    skip_if_not_matrix,
    local_matrix_server,
    private_rooms,
    retry_interval,
    retries_before_backoff,
):
    """ Test the retry mechanism implemented into the matrix client.
    The test creates a transport and sends a message. Given that the
    receiver was online, the initial message is sent but the receiver
    doesn't respond in time and goes offline. The retrier should then
    wait for the `retry_interval` duration to pass and send the message
    again but this won't work because the receiver is offline. Once
    the receiver comes back again, the message should be sent again.
    """
    partner_address = make_address()

    transport = MatrixTransport({
        'discovery_room': 'discovery',
        'retries_before_backoff': retries_before_backoff,
        'retry_interval': retry_interval,
        'server': local_matrix_server,
        'server_name': 'matrix.local.raiden',
        'available_servers': [],
        'private_rooms': private_rooms,
    })
    transport._send_raw = MagicMock()
    raiden_service = MockRaidenService(None)

    transport.start(
        raiden_service,
        raiden_service.message_handler,
        None,
    )
    transport.log = MagicMock()

    # Receiver is online
    transport._address_to_presence[partner_address] = UserPresence.ONLINE

    queueid = QueueIdentifier(
        recipient=partner_address,
        channel_identifier=CHANNEL_IDENTIFIER_GLOBAL_QUEUE,
    )
    chain_state = raiden_service.wal.state_manager.current_state

    retry_queue: _RetryQueue = transport._get_retrier(partner_address)
    assert bool(retry_queue), 'retry_queue not running'

    # Send the initial message
    message = Processed(0)
    message.sign(transport._raiden_service.private_key)
    chain_state.queueids_to_queues[queueid] = [message]
    retry_queue.enqueue_global(message)

    gevent.sleep(1)

    transport._send_raw.call_count = 1

    # Receiver goes offline
    transport._address_to_presence[partner_address] = UserPresence.OFFLINE

    gevent.sleep(retry_interval)

    transport.log.debug.assert_called_with(
        'Partner not reachable. Skipping.',
        partner=pex(partner_address),
        status=UserPresence.OFFLINE,
    )

    # Retrier did not call send_raw given that the receiver is still offline
    assert transport._send_raw.call_count == 1

    # Receiver comes back online
    transport._address_to_presence[partner_address] = UserPresence.ONLINE

    gevent.sleep(retry_interval)

    # Retrier now should have sent the message again
    assert transport._send_raw.call_count == 2

    transport.stop()
    transport.get()
<commit_msg>Add test for invalid server on _join_discovery_room<commit_after>import json
import random
from unittest.mock import MagicMock

import gevent
import pytest

from raiden.constants import UINT64_MAX
from raiden.messages import Processed, SecretRequest
from raiden.network.transport import MatrixTransport
from raiden.network.transport.matrix import UserPresence, _RetryQueue
from raiden.tests.utils.factories import HOP1, HOP1_KEY, UNIT_SECRETHASH, make_address
from raiden.tests.utils.mocks import MockRaidenService
from raiden.transfer.mediated_transfer.events import CHANNEL_IDENTIFIER_GLOBAL_QUEUE
from raiden.transfer.queue_identifier import QueueIdentifier
from raiden.transfer.state_change import ActionUpdateTransportAuthData
from raiden.utils import pex
from raiden.utils.typing import Address, List, Optional, Union
from raiden_libs.network.matrix import Room

USERID1 = '@Alice:Wonderland'

@pytest.fixture
def mock_matrix(
        monkeypatch,
        retry_interval,
        retries_before_backoff,
        local_matrix_server,
        private_rooms,
):

    from matrix_client.user import User
    monkeypatch.setattr(User, 'get_display_name', lambda _: 'random_display_name')

    def mock_get_user(klass, user: Union[User, str]) -> User:
        return User(None, USERID1)

    def mock_get_room_ids_for_address(
            klass,
            address: Address,
            filter_private: bool=None,
    ) -> List[str]:
        return ['!roomID:server']

    def mock_set_room_id_for_address(self, address: Address, room_id: Optional[str]):
        pass

    def mock_receive_message(klass, message):
        # We are just unit testing the matrix transport receive so do nothing
        assert message

    def mock_receive_delivered(klass, delivered):
        # We are just unit testing the matrix transport receive so do nothing
        assert delivered

    config = dict(
        retry_interval=retry_interval,
        retries_before_backoff=retries_before_backoff,
        server=local_matrix_server,
        server_name='matrix.local.raiden',
        available_servers=[],
        discovery_room='discovery',
        private_rooms=private_rooms,
    )

    transport = MatrixTransport(config)
    transport._raiden_service = MockRaidenService()
    transport._stop_event.clear()
    transport._address_to_userids[HOP1] = USERID1

    monkeypatch.setattr(MatrixTransport, '_get_user', mock_get_user)
    monkeypatch.setattr(
        MatrixTransport,
        '_get_room_ids_for_address',
        mock_get_room_ids_for_address,
    )
    monkeypatch.setattr(MatrixTransport, '_set_room_id_for_address', mock_set_room_id_for_address)
    monkeypatch.setattr(MatrixTransport, '_receive_message', mock_receive_message)

    return transport

@pytest.fixture()
def skip_userid_validation(monkeypatch):
    def mock_validate_userid_signature(klass, user):
        return HOP1

    monkeypatch.setattr(
        MatrixTransport,
        '_validate_userid_signature',
        mock_validate_userid_signature,
    )

def make_message(convert_to_hex: bool = False, overwrite_data=None):
    from matrix_client.room import Room
    room = Room(None, '!roomID:server')
    if not overwrite_data:
        message = SecretRequest(
            message_identifier=random.randint(0, UINT64_MAX),
            payment_identifier=1,
            secrethash=UNIT_SECRETHASH,
            amount=1,
            expiration=10,
        )
        message.sign(HOP1_KEY)
        data = message.encode()
        if convert_to_hex:
            data = '0x' + data.hex()
        else:
            data = json.dumps(message.to_dict())
    else:
        data = overwrite_data

    event = dict(
        type='m.room.message',
        sender=USERID1,
        content={
            'msgtype': 'm.text',
            'body': data,
        },
    )
    return room, event

def test_normal_processing_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(convert_to_hex=True)
    assert m._handle_message(room, event)

def test_normal_processing_json(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(convert_to_hex=False)
    assert m._handle_message(room, event)

def test_processing_invalid_json(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    invalid_json = '{"foo": 1,'
    room, event = make_message(convert_to_hex=False, overwrite_data=invalid_json)
    assert not m._handle_message(room, event)

def test_sending_nonstring_body(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(overwrite_data=b'somebinarydata')
    assert not m._handle_message(room, event)

def test_processing_invalid_message_json(
        mock_matrix,
        skip_userid_validation,
        skip_if_not_matrix,
):
    m = mock_matrix
    invalid_message = '{"this": 1, "message": 5, "is": 3, "not_valid": 5}'
    room, event = make_message(convert_to_hex=False, overwrite_data=invalid_message)
    assert not m._handle_message(room, event)

def test_processing_invalid_message_cmdid_json(
        mock_matrix,
        skip_userid_validation,
        skip_if_not_matrix,
):
    m = mock_matrix
    invalid_message = '{"type": "NonExistentMessage", "is": 3, "not_valid": 5}'
    room, event = make_message(convert_to_hex=False, overwrite_data=invalid_message)
    assert not m._handle_message(room, event)

def test_processing_invalid_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(convert_to_hex=True)
    old_data = event['content']['body']
    event['content']['body'] = old_data[:-1]
    assert not m._handle_message(room, event)

def test_processing_invalid_message_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix):
    m = mock_matrix
    room, event = make_message(convert_to_hex=True)
    old_data = event['content']['body']
    event['content']['body'] = old_data[:-4]
    assert not m._handle_message(room, event)

def test_processing_invalid_message_cmdid_hex(
        mock_matrix,
        skip_userid_validation,
        skip_if_not_matrix,
):
    m = mock_matrix
    room, event = make_message(convert_to_hex=True)
    old_data = event['content']['body']
    event['content']['body'] = '0xff' + old_data[4:]
    assert not m._handle_message(room, event)

def test_matrix_message_sync(
        skip_if_not_matrix,
        local_matrix_server,
        private_rooms,
        retry_interval,
        retries_before_backoff,
):
    transport0 = MatrixTransport({
        'discovery_room': 'discovery',
        'retries_before_backoff': retries_before_backoff,
        'retry_interval': retry_interval,
        'server': local_matrix_server,
        'server_name': 'matrix.local.raiden',
        'available_servers': [],
        'private_rooms': private_rooms,
    })
    transport1 = MatrixTransport({
        'discovery_room': 'discovery',
        'retries_before_backoff': retries_before_backoff,
        'retry_interval': retry_interval,
        'server': local_matrix_server,
        'server_name': 'matrix.local.raiden',
        'available_servers': [],
        'private_rooms': private_rooms,
    })

    latest_auth_data = None

    received_messages = set()

    class MessageHandler:
        def on_message(self, _, message):
            nonlocal received_messages
            received_messages.add(message)

    message_handler = MessageHandler()
    raiden_service0 = MockRaidenService(message_handler)
    raiden_service1 = MockRaidenService(message_handler)

    raiden_service1.handle_state_change = MagicMock()

    transport0.start(
        raiden_service0,
        message_handler,
        None,
    )
    transport1.start(
        raiden_service1,
        message_handler,
        None,
    )

    gevent.sleep(1)

    latest_auth_data = f'{transport1._user_id}/{transport1._client.api.token}'
    update_transport_auth_data = ActionUpdateTransportAuthData(latest_auth_data)
    raiden_service1.handle_state_change.assert_called_with(update_transport_auth_data)

    transport0.start_health_check(transport1._raiden_service.address)
    transport1.start_health_check(transport0._raiden_service.address)

    queue_identifier = QueueIdentifier(
        recipient=transport1._raiden_service.address,
        channel_identifier=1,
    )

    for i in range(5):
        message = Processed(i)
        message.sign(transport0._raiden_service.private_key)
        transport0.send_async(
            queue_identifier,
            message,
        )

    gevent.sleep(2)

    assert len(received_messages) == 10
    for i in range(5):
        assert any(getattr(m, 'message_identifier', -1) == i for m in received_messages)

    transport1.stop()

    assert latest_auth_data

    # Send more messages while the other end is offline
    for i in range(10, 15):
        message = Processed(i)
        message.sign(transport0._raiden_service.private_key)
        transport0.send_async(
            queue_identifier,
            message,
        )

    # Should fetch the 5 messages sent while transport1 was offline
    transport1.start(
        transport1._raiden_service,
        message_handler,
        latest_auth_data,
    )

    gevent.sleep(2)

    assert len(set(received_messages)) == 20
    for i in range(10, 15):
        assert any(getattr(m, 'message_identifier', -1) == i for m in received_messages)

    transport0.stop()
    transport1.stop()
    transport0.get()
    transport1.get()

def test_matrix_message_retry(
    skip_if_not_matrix,
    local_matrix_server,
    private_rooms,
    retry_interval,
    retries_before_backoff,
):
    """ Test the retry mechanism implemented into the matrix client.
    The test creates a transport and sends a message. Given that the
    receiver was online, the initial message is sent but the receiver
    doesn't respond in time and goes offline. The retrier should then
    wait for the `retry_interval` duration to pass and send the message
    again but this won't work because the receiver is offline. Once
    the receiver comes back again, the message should be sent again.
    """
    partner_address = make_address()

    transport = MatrixTransport({
        'discovery_room': 'discovery',
        'retries_before_backoff': retries_before_backoff,
        'retry_interval': retry_interval,
        'server': local_matrix_server,
        'server_name': 'matrix.local.raiden',
        'available_servers': [],
        'private_rooms': private_rooms,
    })
    transport._send_raw = MagicMock()
    raiden_service = MockRaidenService(None)

    transport.start(
        raiden_service,
        raiden_service.message_handler,
        None,
    )
    transport.log = MagicMock()

    # Receiver is online
    transport._address_to_presence[partner_address] = UserPresence.ONLINE

    queueid = QueueIdentifier(
        recipient=partner_address,
        channel_identifier=CHANNEL_IDENTIFIER_GLOBAL_QUEUE,
    )
    chain_state = raiden_service.wal.state_manager.current_state

    retry_queue: _RetryQueue = transport._get_retrier(partner_address)
    assert bool(retry_queue), 'retry_queue not running'

    # Send the initial message
    message = Processed(0)
    message.sign(transport._raiden_service.private_key)
    chain_state.queueids_to_queues[queueid] = [message]
    retry_queue.enqueue_global(message)

    gevent.sleep(1)

    transport._send_raw.call_count = 1

    # Receiver goes offline
    transport._address_to_presence[partner_address] = UserPresence.OFFLINE

    gevent.sleep(retry_interval)

    transport.log.debug.assert_called_with(
        'Partner not reachable. Skipping.',
        partner=pex(partner_address),
        status=UserPresence.OFFLINE,
    )

    # Retrier did not call send_raw given that the receiver is still offline
    assert transport._send_raw.call_count == 1

    # Receiver comes back online
    transport._address_to_presence[partner_address] = UserPresence.ONLINE

    gevent.sleep(retry_interval)

    # Retrier now should have sent the message again
    assert transport._send_raw.call_count == 2

    transport.stop()
    transport.get()

def test_join_invalid_discovery(
    skip_if_not_matrix,
    local_matrix_server,
    private_rooms,
    retry_interval,
    retries_before_backoff,
):
    """_join_discovery_room tries to join on all servers on available_servers config

    If any of the servers isn't reachable by synapse, it'll return a 500 response, which needs
    to be handled, and if no discovery room is found on any of the available_servers, one in
    our current server should be created
    """
    transport = MatrixTransport({
        'discovery_room': 'discovery',
        'retries_before_backoff': retries_before_backoff,
        'retry_interval': retry_interval,
        'server': local_matrix_server,
        'server_name': 'matrix.local.raiden',
        'available_servers': ['http://invalid.server'],
        'private_rooms': private_rooms,
    })
    transport._client.api.retry_timeout = 0
    transport._send_raw = MagicMock()
    raiden_service = MockRaidenService(None)

    transport.start(
        raiden_service,
        raiden_service.message_handler,
        None,
    )
    transport.log = MagicMock()

    transport._join_discovery_room()
    assert isinstance(transport._discovery_room, Room)

    transport.stop()
    transport.get()
<|endoftext|>


<commit_before>#!/usr/bin/env python

#
# Copyright 2013, Couchbase, Inc.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file demonstrates some of the functionalities available with
# view queries. This creates a bunch of key-value pairs where the value is
# a number. It also creates a view to index the key-value pairs by the
# number itself, and finally queries the view to return the ten items with
# the highest values.

from argparse import ArgumentParser
import random
import pprint

from couchbase.connection import Connection

ap = ArgumentParser()

ap.add_argument('-D', '--create-design', default=False,
                action='store_true',
                help='whether to create the design')

ap.add_argument('-n', '--number-of-terms', default=10,
                type=int, help="How many terms to generate")

options = ap.parse_args()

c = Connection(bucket='default')

DESIGN = {
    '_id': '_design/search_keywords',
    'language': 'javascript',
    'views': {
        'top_keywords': {
            'map':
            """
            function(doc) {
                if (typeof doc === 'number') {
                    emit(doc, null);
                }
            }
            """
        }
    }
}

if options.create_design:
    c._design('search_keywords', DESIGN)

NOUNS = ['cow', 'cat', 'dog', 'computer', 'WMD']
ADJECTIVES = ['happy', 'sad', 'thoughtful', 'extroverted']

kv = {}

for x in range(options.number_of_terms):
    n = random.choice(NOUNS)
    a = random.choice(ADJECTIVES)
    kv[" ".join([a, n])] = random.randint(1, 100000)

c.set_multi(kv)

vret = c._view('search_keywords', 'top_keywords',
               {'limit': 10, 'descending': True})

pprint.pprint(vret.value['rows'], indent=4)

# Sample output:
#[   {   u'id': u'WMD sad', u'key': 92772, u'value': None},
#    {   u'id': u'WMD thoughtful', u'key': 76222, u'value': None},
#    {   u'id': u'cow happy', u'key': 71984, u'value': None},
#    {   u'id': u'computer sad', u'key': 68849, u'value': None},
#    {   u'id': u'cat thoughtful', u'key': 68417, u'value': None},
#    {   u'id': u'computer thoughtful', u'key': 67518, u'value': None},
#    {   u'id': u'dog thoughtful', u'key': 67350, u'value': None},
#    {   u'id': u'computer extroverted', u'key': 63279, u'value': None},
#    {   u'id': u'cow thoughtful', u'key': 60962, u'value': None},
#    {   u'id': u'cow sad', u'key': 49510, u'value': None}]
<commit_msg>Update search_keywords example to match current API<commit_after>#!/usr/bin/env python

#
# Copyright 2013, Couchbase, Inc.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file demonstrates some of the functionalities available with
# view queries. This creates a bunch of key-value pairs where the value is
# a number. It also creates a view to index the key-value pairs by the
# number itself, and finally queries the view to return the ten items with
# the highest values.

from argparse import ArgumentParser
import random
import pprint

from couchbase.connection import Connection

ap = ArgumentParser()

ap.add_argument('-D', '--create-design', default=False,
                action='store_true',
                help='whether to create the design')

ap.add_argument('-n', '--number-of-terms', default=10,
                type=int, help="How many terms to generate")

options = ap.parse_args()

c = Connection(bucket='default')

DESIGN = {
    '_id': '_design/search_keywords',
    'language': 'javascript',
    'views': {
        'top_keywords': {
            'map':
            """
            function(doc) {
                if (typeof doc === 'number') {
                    emit(doc, null);
                }
            }
            """
        }
    }
}

if options.create_design:
    c.design_create('search_keywords',
                    DESIGN,
                    use_devmode=False,
                    syncwait=5)

NOUNS = ['cow', 'cat', 'dog', 'computer', 'WMD']
ADJECTIVES = ['happy', 'sad', 'thoughtful', 'extroverted']

kv = {}

for x in range(options.number_of_terms):
    n = random.choice(NOUNS)
    a = random.choice(ADJECTIVES)
    kv[" ".join([a, n])] = random.randint(1, 100000)

c.set_multi(kv)

vret = c.query('search_keywords',
               'top_keywords',
               limit=10,
               descending=True)

for row in vret:
    pprint.pprint(row, indent=4)

# Sample output:
#[   {   u'id': u'WMD sad', u'key': 92772, u'value': None},
#    {   u'id': u'WMD thoughtful', u'key': 76222, u'value': None},
#    {   u'id': u'cow happy', u'key': 71984, u'value': None},
#    {   u'id': u'computer sad', u'key': 68849, u'value': None},
#    {   u'id': u'cat thoughtful', u'key': 68417, u'value': None},
#    {   u'id': u'computer thoughtful', u'key': 67518, u'value': None},
#    {   u'id': u'dog thoughtful', u'key': 67350, u'value': None},
#    {   u'id': u'computer extroverted', u'key': 63279, u'value': None},
#    {   u'id': u'cow thoughtful', u'key': 60962, u'value': None},
#    {   u'id': u'cow sad', u'key': 49510, u'value': None}]
<|endoftext|>