<commit_before>"""
Module quicklogging
=======================

(logging with a bit of cozyness)

log wrappers
-------------

    Supplied convenience functions fetch a logger with the name
    of the module from which you're calling them.

.. autofunction:: debug

.. autofunction:: info

.. autofunction:: warning

.. autofunction:: error

.. autofunction:: critical

.. autofunction:: exception

print handlers
---------------

.. autofunction:: catch_prints

.. autofunction:: warn_prints

"""

import sys

from .base import _log_with_level
from .base import get_logger
from .stream_wrapper import StreamWrapper

def debug(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.debug`

    Variadic parameters: see :py:meth:`logging.Logger.debug`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('debug', stackoverhead=stackoverhead, *args, **kwargs)

def info(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.info`

    Variadic parameters: see :py:meth:`logging.Logger.info`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('info', stackoverhead=stackoverhead, *args, **kwargs)

def warning(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.warning`

    Variadic parameters: see :py:meth:`logging.Logger.warning`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('warning', stackoverhead=stackoverhead, *args, **kwargs)

def error(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.error`

    Variadic parameters: see :py:meth:`logging.Logger.error`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('error', stackoverhead=stackoverhead, *args, **kwargs)

def critical(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.critical`

    Variadic parameters: see :py:meth:`logging.Logger.critical`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('critical', stackoverhead=stackoverhead, *args, **kwargs)

def exception(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.exception`

    Variadic parameters: see :py:meth:`logging.Logger.exception`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('exception', stackoverhead=stackoverhead, *args, **kwargs)

_CUR_MODULE = object()

def warn_prints(catch_all=False):
    """Activate warning when print is called

    :param bool catch_all: defaults to False, ie. defaults to only warn about
        current module, ignoring imports"""
    raise NotImplementedError("API is yet to be defined, please help me.")

def catch_prints(
    catch_module=_CUR_MODULE,
    catch_all=False,
    include_children=True,
    logfunc=info,
):
    """configure the print() catching: redirects calls to a logger

    By default, only catches calls to print() from logger

    * named after the calling module
    * children of this logger

    .. note::

        API discussion welcome.

        You anderstand the API is not stable.

    :param string catch_module: include children of logger designed by name
    :param bool catch_all: should catch all print() diregarding where
        they're from ?

        .. warning::

            take care of logging propagation (``Logger.propagate()̀``)

    :param function logfunc: function to use for logging messages

    Possible extension ideas:

        * wrap arbitrary output stream
        * different log functions depending on regex applied on messages
        * make it configurable from config file
        * allow exclusion of specific modules
    """
    wrapper, created = StreamWrapper.get_wrapper(
        'sys.stdout',
        sys.stdout,
        logfunc,
        catch_all=catch_all,
    )
    if created:
        sys.stdout = wrapper

    if catch_module is _CUR_MODULE:
        catch_module = get_logger(stackoverhead=1).name

    wrapper.include_in_catch(catch_module, include_children)
<commit_msg>Apparently py 2.7 requires an enc declaration here<commit_after># coding: utf-8

"""
Module quicklogging
=======================

(logging with a bit of cozyness)

log wrappers
-------------

    Supplied convenience functions fetch a logger with the name
    of the module from which you're calling them.

.. autofunction:: debug

.. autofunction:: info

.. autofunction:: warning

.. autofunction:: error

.. autofunction:: critical

.. autofunction:: exception

print handlers
---------------

.. autofunction:: catch_prints

.. autofunction:: warn_prints

"""

import sys

from .base import _log_with_level
from .base import get_logger
from .stream_wrapper import StreamWrapper

def debug(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.debug`

    Variadic parameters: see :py:meth:`logging.Logger.debug`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('debug', stackoverhead=stackoverhead, *args, **kwargs)

def info(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.info`

    Variadic parameters: see :py:meth:`logging.Logger.info`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('info', stackoverhead=stackoverhead, *args, **kwargs)

def warning(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.warning`

    Variadic parameters: see :py:meth:`logging.Logger.warning`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('warning', stackoverhead=stackoverhead, *args, **kwargs)

def error(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.error`

    Variadic parameters: see :py:meth:`logging.Logger.error`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('error', stackoverhead=stackoverhead, *args, **kwargs)

def critical(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.critical`

    Variadic parameters: see :py:meth:`logging.Logger.critical`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('critical', stackoverhead=stackoverhead, *args, **kwargs)

def exception(*args, **kwargs):
    """wrapper for :py:meth:`logging.Logger.exception`

    Variadic parameters: see :py:meth:`logging.Logger.exception`

    :param int stackoverhead: see :py:func:`quicklogging.base.get_logger`
    """
    stackoverhead = kwargs.pop('stackoverhead', 1)
    _log_with_level('exception', stackoverhead=stackoverhead, *args, **kwargs)

_CUR_MODULE = object()

def warn_prints(catch_all=False):
    """Activate warning when print is called

    :param bool catch_all: defaults to False, ie. defaults to only warn about
        current module, ignoring imports"""
    raise NotImplementedError("API is yet to be defined, please help me.")

def catch_prints(
    catch_module=_CUR_MODULE,
    catch_all=False,
    include_children=True,
    logfunc=info,
):
    """configure the print() catching: redirects calls to a logger

    By default, only catches calls to print() from logger

    * named after the calling module
    * children of this logger

    .. note::

        API discussion welcome.

        You anderstand the API is not stable.

    :param string catch_module: include children of logger designed by name
    :param bool catch_all: should catch all print() diregarding where
        they're from ?

        .. warning::

            take care of logging propagation (``Logger.propagate()̀``)

    :param function logfunc: function to use for logging messages

    Possible extension ideas:

        * wrap arbitrary output stream
        * different log functions depending on regex applied on messages
        * make it configurable from config file
        * allow exclusion of specific modules
    """
    wrapper, created = StreamWrapper.get_wrapper(
        'sys.stdout',
        sys.stdout,
        logfunc,
        catch_all=catch_all,
    )
    if created:
        sys.stdout = wrapper

    if catch_module is _CUR_MODULE:
        catch_module = get_logger(stackoverhead=1).name

    wrapper.include_in_catch(catch_module, include_children)
<|endoftext|>


<commit_before>from cornflake import serializers

from radar.api.serializers.common import QueryPatientField
from radar.api.serializers.forms import FormSerializer, EntrySerializer
from radar.api.views.common import (
    PatientObjectDetailView,
    PatientObjectListView
)
from radar.api.views.generics import (
    ListModelView,
    RetrieveModelView,
    ListCreateModelView,
    RetrieveUpdateDestroyModelView,
    parse_args
)
from radar.models.forms import Entry, Form, GroupForm

class FormRequestSerializer(serializers.Serializer):
    patient = QueryPatientField(required=False)

class FormListView(ListModelView):
    serializer_class = FormSerializer
    model_class = Form

    def filter_query(self, query):
        query = super(FormListView, self).filter_query(query)

        args = parse_args(FormRequestSerializer)

        patient = args['patient']

        # Filter by forms relevant to patient
        if patient:
            group_ids = [x.id for x in patient.groups]

            q = GroupForm.query\
                .filter(GroupForm.group_id.in_(group_ids))\
                .filter(GroupForm.form_id == Form.id)\
                .exists()

            query = query.filter(q)

        return query

class FormDetailView(RetrieveModelView):
    serializer_class = FormSerializer
    model_class = Form

class EntryListView(PatientObjectListView):
    serializer_class = EntrySerializer
    model_class = Entry

# TODO filter by form
class EntryDetailView(PatientObjectDetailView):
    serializer_class = EntrySerializer
    model_class = Entry

def register_views(app):
    app.add_url_rule('/forms', view_func=FormListView.as_view('form_list'))
    app.add_url_rule('/forms/<id>', view_func=FormDetailView.as_view('form_detail'))
    app.add_url_rule('/entries', view_func=EntryListView.as_view('entry_list'))
    app.add_url_rule('/entries/<id>', view_func=EntryDetailView.as_view('entry_detail'))
<commit_msg>Filter entries by form<commit_after>from cornflake import serializers, fields

from radar.api.serializers.common import QueryPatientField
from radar.api.serializers.forms import FormSerializer, EntrySerializer, FormField
from radar.api.views.common import (
    PatientObjectDetailView,
    PatientObjectListView
)
from radar.api.views.generics import (
    ListModelView,
    RetrieveModelView,
    ListCreateModelView,
    RetrieveUpdateDestroyModelView,
    parse_args
)
from radar.models.forms import Entry, Form, GroupForm

class FormRequestSerializer(serializers.Serializer):
    patient = QueryPatientField(required=False)

class EntryRequestSerializer(serializers.Serializer):
    form = fields.IntegerField(required=False)

class FormListView(ListModelView):
    serializer_class = FormSerializer
    model_class = Form

    def filter_query(self, query):
        query = super(FormListView, self).filter_query(query)

        args = parse_args(FormRequestSerializer)

        patient = args['patient']

        # Filter by forms relevant to patient
        if patient is not None:
            group_ids = [x.id for x in patient.groups]

            q = GroupForm.query\
                .filter(GroupForm.group_id.in_(group_ids))\
                .filter(GroupForm.form_id == Form.id)\
                .exists()

            query = query.filter(q)

        return query

class FormDetailView(RetrieveModelView):
    serializer_class = FormSerializer
    model_class = Form

class EntryListView(PatientObjectListView):
    serializer_class = EntrySerializer
    model_class = Entry

    def filter_query(self, query):
        query = super(EntryListView, self).filter_query(query)

        args = parse_args(EntryRequestSerializer)

        form_id = args['form']

        # Filter by entries by form
        if form_id is not None:
            query = query.filter(Entry.form_id == form_id)

        return query

class EntryDetailView(PatientObjectDetailView):
    serializer_class = EntrySerializer
    model_class = Entry

def register_views(app):
    app.add_url_rule('/forms', view_func=FormListView.as_view('form_list'))
    app.add_url_rule('/forms/<id>', view_func=FormDetailView.as_view('form_detail'))
    app.add_url_rule('/entries', view_func=EntryListView.as_view('entry_list'))
    app.add_url_rule('/entries/<id>', view_func=EntryDetailView.as_view('entry_detail'))
<|endoftext|>


<commit_before>from datetime import datetime
from sqlalchemy import Column, Integer, ForeignKey, String, select, join, Date, DateTime
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship, aliased
from radar.database import db
from radar.models import DataSource, CreatedModifiedMixin, PatientMixin
from radar.sda.models import SDAPatient, SDABundle

class Patient(db.Model):
    __tablename__ = 'patients'

    id = Column(Integer, primary_key=True)

    registered_user_id = Column(Integer, ForeignKey('users.id'))
    registered_date = Column(DateTime(timezone=False), default=datetime.now)

    units = relationship('UnitPatient')
    disease_groups = relationship('DiseaseGroupPatient')

    sda_bundles = relationship('SDABundle')

    def _latest_sda_patient_attr(self, attr):
        sda_patient = self.latest_sda_patient

        if sda_patient is None:
            return None

        return getattr(sda_patient, attr)

    @classmethod
    def _latest_sda_patient_query(cls, column):
        patient_alias = aliased(Patient)

        # TODO last updated
        return select([column]) \
            .select_from(join(SDAPatient, SDABundle).join(patient_alias)) \
            .where(patient_alias.id == cls.id) \
            .limit(1)\
            .as_scalar()

    @property
    def latest_sda_patient(self):
        latest_sda_patient = None

        for sda_bundle in self.sda_bundles:
            sda_patient = sda_bundle.sda_patient

            if sda_patient is None:
                continue

            # TODO last updated
            latest_sda_patient = sda_patient
            break

        return latest_sda_patient

    @hybrid_property
    def first_name(self):
        return self._latest_sda_patient_attr('first_name')

    @hybrid_property
    def last_name(self):
        return self._latest_sda_patient_attr('last_name')

    @hybrid_property
    def date_of_birth(self):
        return self._latest_sda_patient_attr('date_of_birth')

    @hybrid_property
    def gender(self):
        return self._latest_sda_patient_attr('gender')

    @hybrid_property
    def is_male(self):
        return self.gender == 'M'

    @hybrid_property
    def is_female(self):
        return self.gender == 'F'

    @first_name.expression
    def first_name(cls):
        return cls._latest_sda_patient_query(SDAPatient.first_name)

    @last_name.expression
    def last_name(cls):
        return cls._latest_sda_patient_query(SDAPatient.last_name)

    @date_of_birth.expression
    def date_of_birth(cls):
        return cls._latest_sda_patient_query(SDAPatient.date_of_birth)

    @gender.expression
    def gender(cls):
        return cls._latest_sda_patient_query(SDAPatient.gender)

    def _has_unit_permission(self, user, permission):
        patient_units = set([x.unit for x in self.units])

        for unit_user in user.units:
            if getattr(unit_user, 'has_' + permission + '_permission') and unit_user.unit in patient_units:
                return True

        return False

    def _has_disease_group_permission(self, user, permission):
        patient_disease_groups = set([x.disease_group for x in self.disease_groups])

        for disease_group_user in user.disease_groups:
            if getattr(disease_group_user, 'has_' + permission + '_permission') and\
                            disease_group_user.disease_group in patient_disease_groups:
                return True

        return False

    def can_view(self, user):
        return (
            user.is_admin or
            self._has_unit_permission(user, 'view_patient') or
            self._has_disease_group_permission(user, 'view_patient')
        )

    def can_edit(self, user):
        return (
           user.is_admin or
           self._has_unit_permission(user, 'edit_patient')
        )

    def can_view_demographics(self, user):
        if user.is_admin:
            return True

        if self._has_unit_permission(user, 'view_demographics'):
            return True

        if self._has_disease_group_permission(user, 'view_demographics'):
            return True

        return False

    def filter_units_for_user(self, user):
        """ Patient units a user can view """

        _ = user
        return self.units

    def filter_disease_groups_for_user(self, user):
        """ Patient disease groups a user can view """

        # The user can view all of the patient's disease groups if:
        # * The user is an admin
        # * The patient belongs to one of the user's units
        if user.is_admin or self._has_unit_permission(user, 'view_patient'):
            return self.disease_groups
        else:
            # Otherwise intersect the disease groups of the patient and the user
            user_disease_groups = set([x.disease_group for x in user.disease_groups])
            common_disease_groups = [x for x in self.disease_groups if x.disease_group in user_disease_groups]
            return common_disease_groups

    def available_units_for_user(self, user):
        """ Patient units that a user can save data under """

        # TODO intersect patient units with user units with write permissions
        return self.units

    def in_disease_group(self, disease_group):
        # TODO
        return False

    def in_unit(self, unit):
        # TODO
        return False

class Demographics(DataSource, PatientMixin, CreatedModifiedMixin):
    __tablename__ = 'demographics'

    id = Column(Integer, ForeignKey('data_sources.id'), primary_key=True)

    first_name = Column(String)
    last_name = Column(String)
    date_of_birth = Column(Date)
    gender = Column(Integer)

    # TODO
    ethnicity = Column(String)

    alias_first_name = Column(String)
    alias_last_name = Column(String)

    address_line_1 = Column(String)
    address_line_2 = Column(String)
    address_line_3 = Column(String)
    postcode = Column(String)

    home_number = Column(String)
    work_number = Column(String)
    mobile_number = Column(String)
    email_address = Column(String)

    nhs_no = Column(Integer)
    chi_no = Column(Integer)

    __mapper_args__ = {
        'polymorphic_identity': 'demographics',
    }

    def can_view(self, user):
        return self.patient.can_view_demographics(user)

    def can_edit(self, user):
        return self.patient.can_edit(user) and self.patient.can_view_demographics(user)<commit_msg>Add column to store which unit registered the patient<commit_after>from datetime import datetime
from sqlalchemy import Column, Integer, ForeignKey, String, select, join, Date, DateTime
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship, aliased
from radar.database import db
from radar.models import DataSource, CreatedModifiedMixin, PatientMixin
from radar.sda.models import SDAPatient, SDABundle

class Patient(db.Model):
    __tablename__ = 'patients'

    id = Column(Integer, primary_key=True)

    registered_user_id = Column(Integer, ForeignKey('users.id'))
    registered_date = Column(DateTime(timezone=False), default=datetime.now)
    registered_unit_id = Column(Integer, ForeignKey('units.id'))

    units = relationship('UnitPatient')
    disease_groups = relationship('DiseaseGroupPatient')

    sda_bundles = relationship('SDABundle')

    def _latest_sda_patient_attr(self, attr):
        sda_patient = self.latest_sda_patient

        if sda_patient is None:
            return None

        return getattr(sda_patient, attr)

    @classmethod
    def _latest_sda_patient_query(cls, column):
        patient_alias = aliased(Patient)

        # TODO last updated
        return select([column]) \
            .select_from(join(SDAPatient, SDABundle).join(patient_alias)) \
            .where(patient_alias.id == cls.id) \
            .limit(1)\
            .as_scalar()

    @property
    def latest_sda_patient(self):
        latest_sda_patient = None

        for sda_bundle in self.sda_bundles:
            sda_patient = sda_bundle.sda_patient

            if sda_patient is None:
                continue

            # TODO last updated
            latest_sda_patient = sda_patient
            break

        return latest_sda_patient

    @hybrid_property
    def first_name(self):
        return self._latest_sda_patient_attr('first_name')

    @hybrid_property
    def last_name(self):
        return self._latest_sda_patient_attr('last_name')

    @hybrid_property
    def date_of_birth(self):
        return self._latest_sda_patient_attr('date_of_birth')

    @hybrid_property
    def gender(self):
        return self._latest_sda_patient_attr('gender')

    @hybrid_property
    def is_male(self):
        return self.gender == 'M'

    @hybrid_property
    def is_female(self):
        return self.gender == 'F'

    @first_name.expression
    def first_name(cls):
        return cls._latest_sda_patient_query(SDAPatient.first_name)

    @last_name.expression
    def last_name(cls):
        return cls._latest_sda_patient_query(SDAPatient.last_name)

    @date_of_birth.expression
    def date_of_birth(cls):
        return cls._latest_sda_patient_query(SDAPatient.date_of_birth)

    @gender.expression
    def gender(cls):
        return cls._latest_sda_patient_query(SDAPatient.gender)

    def _has_unit_permission(self, user, permission):
        patient_units = set([x.unit for x in self.units])

        for unit_user in user.units:
            if getattr(unit_user, 'has_' + permission + '_permission') and unit_user.unit in patient_units:
                return True

        return False

    def _has_disease_group_permission(self, user, permission):
        patient_disease_groups = set([x.disease_group for x in self.disease_groups])

        for disease_group_user in user.disease_groups:
            if getattr(disease_group_user, 'has_' + permission + '_permission') and\
                            disease_group_user.disease_group in patient_disease_groups:
                return True

        return False

    def can_view(self, user):
        return (
            user.is_admin or
            self._has_unit_permission(user, 'view_patient') or
            self._has_disease_group_permission(user, 'view_patient')
        )

    def can_edit(self, user):
        return (
           user.is_admin or
           self._has_unit_permission(user, 'edit_patient')
        )

    def can_view_demographics(self, user):
        if user.is_admin:
            return True

        if self._has_unit_permission(user, 'view_demographics'):
            return True

        if self._has_disease_group_permission(user, 'view_demographics'):
            return True

        return False

    def filter_units_for_user(self, user):
        """ Patient units a user can view """

        _ = user
        return self.units

    def filter_disease_groups_for_user(self, user):
        """ Patient disease groups a user can view """

        # The user can view all of the patient's disease groups if:
        # * The user is an admin
        # * The patient belongs to one of the user's units
        if user.is_admin or self._has_unit_permission(user, 'view_patient'):
            return self.disease_groups
        else:
            # Otherwise intersect the disease groups of the patient and the user
            user_disease_groups = set([x.disease_group for x in user.disease_groups])
            common_disease_groups = [x for x in self.disease_groups if x.disease_group in user_disease_groups]
            return common_disease_groups

    def available_units_for_user(self, user):
        """ Patient units that a user can save data under """

        # TODO intersect patient units with user units with write permissions
        return self.units

    def in_disease_group(self, disease_group):
        # TODO
        return False

    def in_unit(self, unit):
        # TODO
        return False

class Demographics(DataSource, PatientMixin, CreatedModifiedMixin):
    __tablename__ = 'demographics'

    id = Column(Integer, ForeignKey('data_sources.id'), primary_key=True)

    first_name = Column(String)
    last_name = Column(String)
    date_of_birth = Column(Date)
    gender = Column(Integer)

    # TODO
    ethnicity = Column(String)

    alias_first_name = Column(String)
    alias_last_name = Column(String)

    address_line_1 = Column(String)
    address_line_2 = Column(String)
    address_line_3 = Column(String)
    postcode = Column(String)

    home_number = Column(String)
    work_number = Column(String)
    mobile_number = Column(String)
    email_address = Column(String)

    nhs_no = Column(Integer)
    chi_no = Column(Integer)

    __mapper_args__ = {
        'polymorphic_identity': 'demographics',
    }

    def can_view(self, user):
        return self.patient.can_view_demographics(user)

    def can_edit(self, user):
        return self.patient.can_edit(user) and self.patient.can_view_demographics(user)<|endoftext|>


<commit_before>from raggregate.models import DBSession
from raggregate.models.motd import MOTD
from raggregate.new_queries import motd as motd_queries

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPFound
from pyramid.httpexceptions import HTTPNotFound

@view_config(renderer='motd.mak', route_name='motd')
def motd(request):
    session = request.session
    post = session['safe_post']
    dbsession = DBSession()

    # Only allow admins to add a message at this time
    #@TODO: unify admin-only page handling so that we can easily change this
    # some day if we want.
    #if 'logged_in_admin' not in session or session['logged_in_admin'] == False:
    #    return HTTPNotFound()

    # Ensure user is logged in
    if 'logged_in' not in session:
        session['message'] = 'Sorry, you must log in before you can add a message of the day.'.format(request.route_url('login'))
        return {'messages_of_the_day': [], 'success': False, 'code': 'ENOLOGIN'}

    # Handle the form input
    if post and post['message_to_add'] != '':
        message_to_add = post['message_to_add']
        author = post['author']

        #@TODO: Need to add more form validation here
        if author == "":
            author = "Unknown"

        try:
            new_MOTD = MOTD(message = new_motd, author = author, added_by = session['users.id'])
            dbsession.add(new_MOTD)
            dbsession.commit()
            session['message'] = "Message of the Day Added!"
        except Exception, ex:
            print str(ex)

    messages_of_the_day = motd_queries.get_all_messages()
    return {'messages_of_the_day': messages_of_the_day}
<commit_msg>Restrict motd page to admin users only<commit_after>from raggregate.models import DBSession
from raggregate.models.motd import MOTD
from raggregate.new_queries import motd as motd_queries

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPFound
from pyramid.httpexceptions import HTTPNotFound

@view_config(renderer='motd.mak', route_name='motd')
def motd(request):
    session = request.session
    post = session['safe_post']
    dbsession = DBSession()

    # Only allow admins to add a message at this time
    #@TODO: unify admin-only page handling so that we can easily change this
    # some day if we want.
    if 'logged_in_admin' not in session or session['logged_in_admin'] == False:
        return HTTPNotFound()

    # Handle the form input
    if post and post['message_to_add'] != '':
        message_to_add = post['message_to_add']
        author = post['author']

        #@TODO: Need to add more form validation here
        if author == "":
            author = "Unknown"

        try:
            new_MOTD = MOTD(message = new_motd, author = author, added_by = session['users.id'])
            dbsession.add(new_MOTD)
            dbsession.commit()
            session['message'] = "Message of the Day Added!"
        except Exception, ex:
            print str(ex)

    messages_of_the_day = motd_queries.get_all_messages()
    return {'messages_of_the_day': messages_of_the_day}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# pylint: disable=wrong-import-position,redefined-outer-name,unused-wildcard-import,wildcard-import
import re

import gevent
import py
import sys
from gevent import monkey
monkey.patch_all()

import pytest

from raiden.exceptions import RaidenShuttingDown
from raiden.tests.fixtures.variables import *  # noqa: F401,F403
from raiden.log_config import configure_logging

CATCH_LOG_HANDLER_NAME = 'catch_log_handler'

def pytest_addoption(parser):
    parser.addoption(
        '--blockchain-type',
        choices=['geth'],
        default='geth',
    )

    parser.addoption(
        '--initial-port',
        type=int,
        default=29870,
        help='Base port number used to avoid conflicts while running parallel tests.',
    )

    parser.addoption(
        '--log-config',
        default=None,
    )

    parser.addoption(
        '--plain-log',
        action='store_true',
        default=False,
        help='Do not colorize console log output'
    )
    parser.addoption(
        '--profiler',
        default=None,
        choices=['cpu', 'sample'],
    )

@pytest.fixture(autouse=True)
def profiler(request, tmpdir):
    if request.config.option.profiler == 'cpu':
        from raiden.utils.profiling.cpu import CpuProfiler
        profiler = CpuProfiler(str(tmpdir))
        profiler.start()

        yield

        profiler.stop()

    elif request.config.option.profiler == 'sample':
        from raiden.utils.profiling.sampler import SampleProfiler
        profiler = SampleProfiler(str(tmpdir))
        profiler.start()

        yield

        profiler.stop()

    else:
        # do nothing, but yield a valid generator otherwise the autouse fixture
        # will fail
        yield

@pytest.fixture(autouse=True, scope='session')
def logging_level(request):
    """ Configure the structlog level.

    For integration tests this also sets the geth verbosity.
    """
    if request.config.option.verbose > 3:
        level = 'DEBUG'
    elif request.config.option.verbose > 1:
        level = 'INFO'
    else:
        level = 'WARNING'
    if request.config.option.log_cli_level:
        level = request.config.option.log_cli_level
    configure_logging(
        {'': level},
        colorize=not request.config.option.plain_log,
        log_file=request.config.option.log_file
    )

@pytest.fixture(scope='session', autouse=True)
def enable_greenlet_debugger(request):
    if request.config.option.usepdb:
        from raiden.utils.debug import enable_greenlet_debugger
        enable_greenlet_debugger()

@pytest.fixture(scope='session', autouse=True)
def validate_solidity_compiler():
    """ Check the solc prior to running any test. """
    from raiden.blockchain.abi import validate_solc
    validate_solc()

@pytest.fixture(scope='session', autouse=True)
def dont_exit_pytest():
    """ Raiden will quit on any unhandled exception.

    This allows the test suite to finish in case an exception is unhandled.
    """
    gevent.get_hub().SYSTEM_ERROR = BaseException
    gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit, RaidenShuttingDown)

if sys.platform == 'darwin':
    # On macOS the temp directory base path is already very long.
    # To avoid failures on ipc tests (ipc path length is limited to 104/108 chars on macOS/linux)
    # we override the pytest tmpdir machinery to produce shorter paths.

    @pytest.fixture(scope='session', autouse=True)
    def _tmpdir_short(request):
        """Shorten tmpdir paths"""
        from _pytest.tmpdir import TempdirFactory

        def getbasetemp(self):
            """ return base temporary directory. """
            try:
                return self._basetemp
            except AttributeError:
                basetemp = self.config.option.basetemp
                if basetemp:
                    basetemp = py.path.local(basetemp)
                    if basetemp.check():
                        basetemp.remove()
                    basetemp.mkdir()
                else:
                    rootdir = py.path.local.get_temproot()
                    rootdir.ensure(dir=1)
                    basetemp = py.path.local.make_numbered_dir(prefix='pyt', rootdir=rootdir)
                self._basetemp = t = basetemp.realpath()
                self.trace('new basetemp', t)
                return t

        TempdirFactory.getbasetemp = getbasetemp
        try:
            delattr(request.config._tmpdirhandler, '_basetemp')
        except AttributeError:
            pass

    @pytest.fixture
    def tmpdir(request, tmpdir_factory):
        """Return a temporary directory path object
        which is unique to each test function invocation,
        created as a sub directory of the base temporary
        directory.  The returned object is a `py.path.local`_
        path object.
        """
        name = request.node.name
        name = re.sub(r'[\W]', '_', name)
        MAXVAL = 15
        if len(name) > MAXVAL:
            name = name[:MAXVAL]
        return tmpdir_factory.mktemp(name, numbered=True)
<commit_msg>readded log-config option for the tests<commit_after># -*- coding: utf-8 -*-
# pylint: disable=wrong-import-position,redefined-outer-name,unused-wildcard-import,wildcard-import
import re

import gevent
import py
import sys
from gevent import monkey
monkey.patch_all()

import pytest

from raiden.utils.cli import LogLevelConfigType
from raiden.exceptions import RaidenShuttingDown
from raiden.tests.fixtures.variables import *  # noqa: F401,F403
from raiden.log_config import configure_logging

CATCH_LOG_HANDLER_NAME = 'catch_log_handler'

def pytest_addoption(parser):
    parser.addoption(
        '--blockchain-type',
        choices=['geth'],
        default='geth',
    )

    parser.addoption(
        '--initial-port',
        type=int,
        default=29870,
        help='Base port number used to avoid conflicts while running parallel tests.',
    )

    parser.addoption(
        '--log-config',
        action='store',
        default=None,
        help='Configure tests log output',
    )

    parser.addoption(
        '--plain-log',
        action='store_true',
        default=False,
        help='Do not colorize console log output',
    )
    parser.addoption(
        '--profiler',
        default=None,
        choices=['cpu', 'sample'],
    )

@pytest.fixture(autouse=True)
def profiler(request, tmpdir):
    if request.config.option.profiler == 'cpu':
        from raiden.utils.profiling.cpu import CpuProfiler
        profiler = CpuProfiler(str(tmpdir))
        profiler.start()

        yield

        profiler.stop()

    elif request.config.option.profiler == 'sample':
        from raiden.utils.profiling.sampler import SampleProfiler
        profiler = SampleProfiler(str(tmpdir))
        profiler.start()

        yield

        profiler.stop()

    else:
        # do nothing, but yield a valid generator otherwise the autouse fixture
        # will fail
        yield

@pytest.fixture(autouse=True, scope='session')
def logging_level(request):
    """ Configure the structlog level.

    For integration tests this also sets the geth verbosity.
    """
    if request.config.option.log_cli_level:
        level = request.config.option.log_cli_level
    elif request.config.option.verbose > 3:
        level = 'DEBUG'
    elif request.config.option.verbose > 1:
        level = 'INFO'
    else:
        level = 'WARNING'

    if request.config.option.log_config:
        config_converter = LogLevelConfigType()
        logging_levels = config_converter.convert(
            value=request.config.option.log_config,
            param=None,
            ctx=None,
        )
    else:
        logging_levels = {'': level}

    configure_logging(
        logging_levels,
        colorize=not request.config.option.plain_log,
        log_file=request.config.option.log_file
    )

@pytest.fixture(scope='session', autouse=True)
def enable_greenlet_debugger(request):
    if request.config.option.usepdb:
        from raiden.utils.debug import enable_greenlet_debugger
        enable_greenlet_debugger()

@pytest.fixture(scope='session', autouse=True)
def validate_solidity_compiler():
    """ Check the solc prior to running any test. """
    from raiden.blockchain.abi import validate_solc
    validate_solc()

@pytest.fixture(scope='session', autouse=True)
def dont_exit_pytest():
    """ Raiden will quit on any unhandled exception.

    This allows the test suite to finish in case an exception is unhandled.
    """
    gevent.get_hub().SYSTEM_ERROR = BaseException
    gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit, RaidenShuttingDown)

if sys.platform == 'darwin':
    # On macOS the temp directory base path is already very long.
    # To avoid failures on ipc tests (ipc path length is limited to 104/108 chars on macOS/linux)
    # we override the pytest tmpdir machinery to produce shorter paths.

    @pytest.fixture(scope='session', autouse=True)
    def _tmpdir_short(request):
        """Shorten tmpdir paths"""
        from _pytest.tmpdir import TempdirFactory

        def getbasetemp(self):
            """ return base temporary directory. """
            try:
                return self._basetemp
            except AttributeError:
                basetemp = self.config.option.basetemp
                if basetemp:
                    basetemp = py.path.local(basetemp)
                    if basetemp.check():
                        basetemp.remove()
                    basetemp.mkdir()
                else:
                    rootdir = py.path.local.get_temproot()
                    rootdir.ensure(dir=1)
                    basetemp = py.path.local.make_numbered_dir(prefix='pyt', rootdir=rootdir)
                self._basetemp = t = basetemp.realpath()
                self.trace('new basetemp', t)
                return t

        TempdirFactory.getbasetemp = getbasetemp
        try:
            delattr(request.config._tmpdirhandler, '_basetemp')
        except AttributeError:
            pass

    @pytest.fixture
    def tmpdir(request, tmpdir_factory):
        """Return a temporary directory path object
        which is unique to each test function invocation,
        created as a sub directory of the base temporary
        directory.  The returned object is a `py.path.local`_
        path object.
        """
        name = request.node.name
        name = re.sub(r'[\W]', '_', name)
        MAXVAL = 15
        if len(name) > MAXVAL:
            name = name[:MAXVAL]
        return tmpdir_factory.mktemp(name, numbered=True)
<|endoftext|>


<commit_before>"""This module defines class dj.AutoPopulate"""
import logging
import datetime
import traceback
import random
import inspect
from tqdm import tqdm
from .expression import QueryExpression, AndList
from .errors import DataJointError, LostConnectionError
import signal
import multiprocessing as mp

# noinspection PyExceptionInherit,PyCallingNonCallable

logger = logging.getLogger(__name__)

# --- helper functions for multiprocessing --

def _initialize_populate(table, jobs, populate_kwargs):
    """
    Initialize the process for mulitprocessing.
    Saves the unpickled copy of the table to the current process and reconnects.
    """
    process = mp.current_process()
    process.table = table
    process.jobs = jobs
    process.populate_kwargs = populate_kwargs
    table.connection.connect()  # reconnect

def _call_populate1(key):
    """
    Call current process' table._populate1()
    :key - a dict specifying job to compute
    :return: key, error if error, otherwise None
    """
    process = mp.current_process()
    return process.table._populate1(key, process.jobs, **process.populate_kwargs)

class AutoPopulate:
    """
    AutoPopulate is a mixin class that adds the method populate() to a Relation class.
    Auto-populated relations must inherit from both Relation and AutoPopulate,
    must define the property `key_source`, and must define the callback method `make`.
    """
    _key_source = None
    _allow_insert = False

    @property
    def key_source(self):
        """
        :return: the query expression that yields primary key values to be passed,
        sequentially, to the ``make`` method when populate() is called.
        The default value is the join of the parent tables references from the primary key.
        Subclasses may override they key_source to change the scope or the granularity
        of the make calls.
        """
        def _rename_attributes(table, props):
            return (table.proj(
                **{attr: ref for attr, ref in props['attr_map'].items() if attr != ref})
                if props['aliased'] else table.proj())

        if self._key_source is None:
            parents = self.target.parents(
                primary=True, as_objects=True, foreign_key_info=True)
            if not parents:
                raise DataJointError('A table must have dependencies '
                                     'from its primary key for auto-populate to work')
            self._key_source = _rename_attributes(*parents[0])
            for q in parents[1:]:
                self._key_source *= _rename_attributes(*q)
        return self._key_source

    def make(self, key):
        """
        Derived classes must implement method `make` that fetches data from tables
        above them in the dependency hierarchy, restricting by the given key,
        computes secondary attributes, and inserts the new tuples into self.
        """
        raise NotImplementedError(
            'Subclasses of AutoPopulate must implement the method `make`')

    @property
    def target(self):
        """
        :return: table to be populated.
        In the typical case, dj.AutoPopulate is mixed into a dj.Table class by
        inheritance and the target is self.
        """
        return self

    def _job_key(self, key):
        """
        :param key:  they key returned for the job from the key source
        :return: the dict to use to generate the job reservation hash
        This method allows subclasses to control the job reservation granularity.
        """
        return key

    def _jobs_to_do(self, restrictions):
        """
        :return: the relation containing the keys to be computed (derived from self.key_source)
        """
        if self.restriction:
            raise DataJointError('Cannot call populate on a restricted table. '
                                 'Instead, pass conditions to populate() as arguments.')
        todo = self.key_source

        # key_source is a QueryExpression subclass -- trigger instantiation
        if inspect.isclass(todo) and issubclass(todo, QueryExpression):
            todo = todo()

        if not isinstance(todo, QueryExpression):
            raise DataJointError('Invalid key_source value')

        try:
            # check if target lacks any attributes from the primary key of key_source
            raise DataJointError(
                'The populate target lacks attribute %s '
                'from the primary key of key_source' % next(
                    name for name in todo.heading.primary_key
                    if name not in self.target.heading))
        except StopIteration:
            pass
        return (todo & AndList(restrictions)).proj()

    def populate(self, *restrictions, suppress_errors=False, return_exception_objects=False,
                 reserve_jobs=False, order="original", limit=None, max_calls=None,
                 display_progress=False, processes=1, make_kwargs=None):
        """
        ``table.populate()`` calls ``table.make(key)`` for every primary key in
        ``self.key_source`` for which there is not already a tuple in table.

        :param restrictions: a list of restrictions each restrict
            (table.key_source - target.proj())
        :param suppress_errors: if True, do not terminate execution.
        :param return_exception_objects: return error objects instead of just error messages
        :param reserve_jobs: if True, reserve jobs to populate in asynchronous fashion
        :param order: "original"|"reverse"|"random"  - the order of execution
        :param limit: if not None, check at most this many keys
        :param max_calls: if not None, populate at most this many keys
        :param display_progress: if True, report progress_bar
        :param processes: number of processes to use. When set to a large number, then
            uses as many as CPU cores
        :param make_kwargs: Keyword arguments which do not affect the result of computation
            to be passed down to each ``make()`` call. Computation arguments should be
            specified within the pipeline e.g. using a `dj.Lookup` table.
        :type make_kwargs: dict, optional
        """
        if self.connection.in_transaction:
            raise DataJointError('Populate cannot be called during a transaction.')

        valid_order = ['original', 'reverse', 'random']
        if order not in valid_order:
            raise DataJointError('The order argument must be one of %s' % str(valid_order))
        jobs = self.connection.schemas[self.target.database].jobs if reserve_jobs else None

        # define and set up signal handler for SIGTERM:
        if reserve_jobs:
            def handler(signum, frame):
                logger.info('Populate terminated by SIGTERM')
                raise SystemExit('SIGTERM received')
            old_handler = signal.signal(signal.SIGTERM, handler)

        keys = (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit)
        if order == "reverse":
            keys.reverse()
        elif order == "random":
            random.shuffle(keys)

        logger.info('Found %d keys to populate' % len(keys))

        keys = keys[:max_calls]
        nkeys = len(keys)

        if processes > 1:
            processes = min(processes, nkeys, mp.cpu_count())

        error_list = []
        populate_kwargs = dict(
            suppress_errors=suppress_errors,
            return_exception_objects=return_exception_objects,
            make_kwargs=make_kwargs)

        if processes == 1:
            for key in tqdm(keys, desc=self.__class__.__name__) if display_progress else keys:
                error = self._populate1(key, jobs, **populate_kwargs)
                if error is not None:
                    error_list.append(error)
        else:
            # spawn multiple processes
            self.connection.close()  # disconnect parent process from MySQL server
            del self.connection._conn.ctx  # SSLContext is not pickleable
            with mp.Pool(processes, _initialize_populate, (self, populate_kwargs)) as pool:
                if display_progress:
                    with tqdm(desc="Processes: ", total=nkeys) as pbar:
                        for error in pool.imap(_call_populate1, keys, chunksize=1):
                            if error is not None:
                                error_list.append(error)
                            pbar.update()
                else:
                    for error in pool.imap(_call_populate1, keys):
                        if error is not None:
                            error_list.append(error)
            self.connection.connect()  # reconnect parent process to MySQL server

        # restore original signal handler:
        if reserve_jobs:
            signal.signal(signal.SIGTERM, old_handler)

        if suppress_errors:
            return error_list

    def _populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs=None):
        """
        populates table for one source key, calling self.make inside a transaction.
        :param jobs: the jobs table or None if not reserve_jobs
        :param key: dict specifying job to populate
        :param suppress_errors: bool if errors should be suppressed and returned
        :param return_exception_objects: if True, errors must be returned as objects
        :return: (key, error) when suppress_errors=True, otherwise None
        """
        make = self._make_tuples if hasattr(self, '_make_tuples') else self.make

        if jobs is None or jobs.reserve(self.target.table_name, self._job_key(key)):
            self.connection.start_transaction()
            if key in self.target:  # already populated
                self.connection.cancel_transaction()
                if jobs is not None:
                    jobs.complete(self.target.table_name, self._job_key(key))
            else:
                logger.info('Populating: ' + str(key))
                self.__class__._allow_insert = True
                try:
                    make(dict(key), **(make_kwargs or {}))
                except (KeyboardInterrupt, SystemExit, Exception) as error:
                    try:
                        self.connection.cancel_transaction()
                    except LostConnectionError:
                        pass
                    error_message = '{exception}{msg}'.format(
                        exception=error.__class__.__name__,
                        msg=': ' + str(error) if str(error) else '')
                    if jobs is not None:
                        # show error name and error message (if any)
                        jobs.error(
                            self.target.table_name, self._job_key(key),
                            error_message=error_message, error_stack=traceback.format_exc())
                    if not suppress_errors or isinstance(error, SystemExit):
                        raise
                    else:
                        logger.error(error)
                        return key, error if return_exception_objects else error_message
                else:
                    self.connection.commit_transaction()
                    if jobs is not None:
                        jobs.complete(self.target.table_name, self._job_key(key))
                finally:
                    self.__class__._allow_insert = False

    def progress(self, *restrictions, display=True):
        """
        Report the progress of populating the table.
        :return: (remaining, total) -- numbers of tuples to be populated
        """
        todo = self._jobs_to_do(restrictions)
        total = len(todo)
        remaining = len(todo - self.target)
        if display:
            print('%-20s' % self.__class__.__name__,
                  'Completed %d of %d (%2.1f%%)   %s' % (
                      total - remaining, total, 100 - 100 * remaining / (total+1e-12),
                      datetime.datetime.strftime(datetime.datetime.now(),
                                                 '%Y-%m-%d %H:%M:%S')), flush=True)
        return remaining, total
<commit_msg>Fix missing jobs arg for _initialize_populate in Pool<commit_after>"""This module defines class dj.AutoPopulate"""
import logging
import datetime
import traceback
import random
import inspect
from tqdm import tqdm
from .expression import QueryExpression, AndList
from .errors import DataJointError, LostConnectionError
import signal
import multiprocessing as mp

# noinspection PyExceptionInherit,PyCallingNonCallable

logger = logging.getLogger(__name__)

# --- helper functions for multiprocessing --

def _initialize_populate(table, jobs, populate_kwargs):
    """
    Initialize the process for mulitprocessing.
    Saves the unpickled copy of the table to the current process and reconnects.
    """
    process = mp.current_process()
    process.table = table
    process.jobs = jobs
    process.populate_kwargs = populate_kwargs
    table.connection.connect()  # reconnect

def _call_populate1(key):
    """
    Call current process' table._populate1()
    :key - a dict specifying job to compute
    :return: key, error if error, otherwise None
    """
    process = mp.current_process()
    return process.table._populate1(key, process.jobs, **process.populate_kwargs)

class AutoPopulate:
    """
    AutoPopulate is a mixin class that adds the method populate() to a Relation class.
    Auto-populated relations must inherit from both Relation and AutoPopulate,
    must define the property `key_source`, and must define the callback method `make`.
    """
    _key_source = None
    _allow_insert = False

    @property
    def key_source(self):
        """
        :return: the query expression that yields primary key values to be passed,
        sequentially, to the ``make`` method when populate() is called.
        The default value is the join of the parent tables references from the primary key.
        Subclasses may override they key_source to change the scope or the granularity
        of the make calls.
        """
        def _rename_attributes(table, props):
            return (table.proj(
                **{attr: ref for attr, ref in props['attr_map'].items() if attr != ref})
                if props['aliased'] else table.proj())

        if self._key_source is None:
            parents = self.target.parents(
                primary=True, as_objects=True, foreign_key_info=True)
            if not parents:
                raise DataJointError('A table must have dependencies '
                                     'from its primary key for auto-populate to work')
            self._key_source = _rename_attributes(*parents[0])
            for q in parents[1:]:
                self._key_source *= _rename_attributes(*q)
        return self._key_source

    def make(self, key):
        """
        Derived classes must implement method `make` that fetches data from tables
        above them in the dependency hierarchy, restricting by the given key,
        computes secondary attributes, and inserts the new tuples into self.
        """
        raise NotImplementedError(
            'Subclasses of AutoPopulate must implement the method `make`')

    @property
    def target(self):
        """
        :return: table to be populated.
        In the typical case, dj.AutoPopulate is mixed into a dj.Table class by
        inheritance and the target is self.
        """
        return self

    def _job_key(self, key):
        """
        :param key:  they key returned for the job from the key source
        :return: the dict to use to generate the job reservation hash
        This method allows subclasses to control the job reservation granularity.
        """
        return key

    def _jobs_to_do(self, restrictions):
        """
        :return: the relation containing the keys to be computed (derived from self.key_source)
        """
        if self.restriction:
            raise DataJointError('Cannot call populate on a restricted table. '
                                 'Instead, pass conditions to populate() as arguments.')
        todo = self.key_source

        # key_source is a QueryExpression subclass -- trigger instantiation
        if inspect.isclass(todo) and issubclass(todo, QueryExpression):
            todo = todo()

        if not isinstance(todo, QueryExpression):
            raise DataJointError('Invalid key_source value')

        try:
            # check if target lacks any attributes from the primary key of key_source
            raise DataJointError(
                'The populate target lacks attribute %s '
                'from the primary key of key_source' % next(
                    name for name in todo.heading.primary_key
                    if name not in self.target.heading))
        except StopIteration:
            pass
        return (todo & AndList(restrictions)).proj()

    def populate(self, *restrictions, suppress_errors=False, return_exception_objects=False,
                 reserve_jobs=False, order="original", limit=None, max_calls=None,
                 display_progress=False, processes=1, make_kwargs=None):
        """
        ``table.populate()`` calls ``table.make(key)`` for every primary key in
        ``self.key_source`` for which there is not already a tuple in table.

        :param restrictions: a list of restrictions each restrict
            (table.key_source - target.proj())
        :param suppress_errors: if True, do not terminate execution.
        :param return_exception_objects: return error objects instead of just error messages
        :param reserve_jobs: if True, reserve jobs to populate in asynchronous fashion
        :param order: "original"|"reverse"|"random"  - the order of execution
        :param limit: if not None, check at most this many keys
        :param max_calls: if not None, populate at most this many keys
        :param display_progress: if True, report progress_bar
        :param processes: number of processes to use. When set to a large number, then
            uses as many as CPU cores
        :param make_kwargs: Keyword arguments which do not affect the result of computation
            to be passed down to each ``make()`` call. Computation arguments should be
            specified within the pipeline e.g. using a `dj.Lookup` table.
        :type make_kwargs: dict, optional
        """
        if self.connection.in_transaction:
            raise DataJointError('Populate cannot be called during a transaction.')

        valid_order = ['original', 'reverse', 'random']
        if order not in valid_order:
            raise DataJointError('The order argument must be one of %s' % str(valid_order))
        jobs = self.connection.schemas[self.target.database].jobs if reserve_jobs else None

        # define and set up signal handler for SIGTERM:
        if reserve_jobs:
            def handler(signum, frame):
                logger.info('Populate terminated by SIGTERM')
                raise SystemExit('SIGTERM received')
            old_handler = signal.signal(signal.SIGTERM, handler)

        keys = (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit)
        if order == "reverse":
            keys.reverse()
        elif order == "random":
            random.shuffle(keys)

        logger.info('Found %d keys to populate' % len(keys))

        keys = keys[:max_calls]
        nkeys = len(keys)

        if processes > 1:
            processes = min(processes, nkeys, mp.cpu_count())

        error_list = []
        populate_kwargs = dict(
            suppress_errors=suppress_errors,
            return_exception_objects=return_exception_objects,
            make_kwargs=make_kwargs)

        if processes == 1:
            for key in tqdm(keys, desc=self.__class__.__name__) if display_progress else keys:
                error = self._populate1(key, jobs, **populate_kwargs)
                if error is not None:
                    error_list.append(error)
        else:
            # spawn multiple processes
            self.connection.close()  # disconnect parent process from MySQL server
            del self.connection._conn.ctx  # SSLContext is not pickleable
            with mp.Pool(processes, _initialize_populate, (self, jobs, populate_kwargs)) as pool:
                if display_progress:
                    with tqdm(desc="Processes: ", total=nkeys) as pbar:
                        for error in pool.imap(_call_populate1, keys, chunksize=1):
                            if error is not None:
                                error_list.append(error)
                            pbar.update()
                else:
                    for error in pool.imap(_call_populate1, keys):
                        if error is not None:
                            error_list.append(error)
            self.connection.connect()  # reconnect parent process to MySQL server

        # restore original signal handler:
        if reserve_jobs:
            signal.signal(signal.SIGTERM, old_handler)

        if suppress_errors:
            return error_list

    def _populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs=None):
        """
        populates table for one source key, calling self.make inside a transaction.
        :param jobs: the jobs table or None if not reserve_jobs
        :param key: dict specifying job to populate
        :param suppress_errors: bool if errors should be suppressed and returned
        :param return_exception_objects: if True, errors must be returned as objects
        :return: (key, error) when suppress_errors=True, otherwise None
        """
        make = self._make_tuples if hasattr(self, '_make_tuples') else self.make

        if jobs is None or jobs.reserve(self.target.table_name, self._job_key(key)):
            self.connection.start_transaction()
            if key in self.target:  # already populated
                self.connection.cancel_transaction()
                if jobs is not None:
                    jobs.complete(self.target.table_name, self._job_key(key))
            else:
                logger.info('Populating: ' + str(key))
                self.__class__._allow_insert = True
                try:
                    make(dict(key), **(make_kwargs or {}))
                except (KeyboardInterrupt, SystemExit, Exception) as error:
                    try:
                        self.connection.cancel_transaction()
                    except LostConnectionError:
                        pass
                    error_message = '{exception}{msg}'.format(
                        exception=error.__class__.__name__,
                        msg=': ' + str(error) if str(error) else '')
                    if jobs is not None:
                        # show error name and error message (if any)
                        jobs.error(
                            self.target.table_name, self._job_key(key),
                            error_message=error_message, error_stack=traceback.format_exc())
                    if not suppress_errors or isinstance(error, SystemExit):
                        raise
                    else:
                        logger.error(error)
                        return key, error if return_exception_objects else error_message
                else:
                    self.connection.commit_transaction()
                    if jobs is not None:
                        jobs.complete(self.target.table_name, self._job_key(key))
                finally:
                    self.__class__._allow_insert = False

    def progress(self, *restrictions, display=True):
        """
        Report the progress of populating the table.
        :return: (remaining, total) -- numbers of tuples to be populated
        """
        todo = self._jobs_to_do(restrictions)
        total = len(todo)
        remaining = len(todo - self.target)
        if display:
            print('%-20s' % self.__class__.__name__,
                  'Completed %d of %d (%2.1f%%)   %s' % (
                      total - remaining, total, 100 - 100 * remaining / (total+1e-12),
                      datetime.datetime.strftime(datetime.datetime.now(),
                                                 '%Y-%m-%d %H:%M:%S')), flush=True)
        return remaining, total
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
import time
import urllib
import urllib.error
import urllib.request
import argparse
import datetime
from datetime import timedelta

DEFAULT_URL = "http://104.236.104.117"
DEFAULT_DELAY = 15

_now = datetime.datetime.now

class DataservClientException(Exception):
    pass

class AddressAlreadyRegistered(DataservClientException):

    def __init__(self, address, url):
        msg = "Address {0} already registered at {1}!".format(address, url)
        super(AddressAlreadyRegistered, self).__init__(msg)

class FarmerNotFound(DataservClientException):

    def __init__(self, url):
        msg = "Farmer not found at {0}!".format(url)
        super(FarmerNotFound, self).__init__(msg)

class FarmerError(DataservClientException):

    def __init__(self, url):
        msg = "Farmer error at {0}!".format(url)
        super(FarmerError, self).__init__(msg)

class InvalidAddress(DataservClientException):

    def __init__(self, address):
        msg = "Address {0} not valid!".format(address)
        super(InvalidAddress, self).__init__(msg)

class ConnectionError(DataservClientException):

    def __init__(self, url):
        msg = "Could not connect to server {0}!".format(url)
        super(ConnectionError, self).__init__(msg)

def register(address, url=DEFAULT_URL):
    """Attempt to register the config address."""

    try:
        api_call = "{0}/api/register/{1}".format(url, address)
        response = urllib.request.urlopen(api_call)
        if response.code == 200:
            print("Address {0} now registered on {1}.".format(address, url))
            return True

    except urllib.error.HTTPError as e:
        if e.code == 409:
            raise AddressAlreadyRegistered(address, url)
        elif e.code == 404:
            raise FarmerNotFound(url)
        elif e.code == 400:
            raise InvalidAddress(address)
        elif e.code == 500:
            raise FarmerError(url)
        else:
            raise e
    except urllib.error.URLError:
        raise ConnectionError(url)

def ping(address, url=DEFAULT_URL):
    """Attempt keep-alive with the server."""
    try:
        print("Pinging {0} with address {1}.".format(url, address))
        api_call = "{0}/api/ping/{1}".format(url, address)
        urllib.request.urlopen(api_call)
        return True

    except urllib.error.HTTPError as e:
        if e.code == 400:
            raise InvalidAddress(address)
        elif e.code == 404:
            raise FarmerNotFound(url)
        elif e.code == 500:
            raise FarmerError(url)
        else:
            raise e
    except urllib.error.URLError:
        raise ConnectionError(url)

def poll(address, register_address=False, url=DEFAULT_URL,
         delay=DEFAULT_DELAY, limit=None):
    """TODO doc string"""
    stop_time = _now() + datetime.timedelta(seconds=limit) if limit else None

    if(register_address):
        register(address)

    while True:
        ping(address)
        if stop_time and _now() >= stop_time:
            return True
        time.sleep(delay)

def _parse_args():
    class ArgumentParser(argparse.ArgumentParser):
        def error(self, message):
            sys.stderr.write('error: %s\n' % message)
            self.print_help()
            sys.exit(2)

    # setup parser
    description = "Dataserve client command-line interface."
    parser = ArgumentParser(description=description)
    command_parser = parser.add_subparsers(
        title='commands', dest='command', metavar="<command>"
    )

    # register command
    register_parser = command_parser.add_parser(
        "register",
        help="Register a bitcoin address with farmer."
    )
    register_parser.add_argument(
        "address",
        help="Required bitcoin address."
    )
    register_parser.add_argument(
        "--url", default=DEFAULT_URL,
        help="Url of the farmer (default: {0}).".format(DEFAULT_URL)
    )

    # ping command
    ping_parser = command_parser.add_parser(
        "ping",
        help="Ping farmer with given address."
    )
    ping_parser.add_argument(
        "address",
        help="Required bitcoin address."
    )
    ping_parser.add_argument(
        "--url", default=DEFAULT_URL,
        help="Url of the farmer (default: {0}).".format(DEFAULT_URL)
    )

    # poll command
    poll_parser = command_parser.add_parser(
        "poll",
        help="Continuously ping farmer with given address."
    )
    poll_parser.add_argument(
        "address",
        help="Required bitcoin address."
    )
    poll_parser.add_argument(
        "--url", default=DEFAULT_URL,
        help="Url of the farmer (default: {0}).".format(DEFAULT_URL)
    )
    poll_parser.add_argument(
        "--delay", default=DEFAULT_DELAY,
        help="Deley between each ping."
    )
    poll_parser.add_argument(
        '--register', action='store_true',
        help="Register address before polling."
    )

    # get values
    arguments = vars(parser.parse_args())
    command_name = arguments.pop("command")
    if not command_name:
        parser.error("No command given!")
    return command_name, arguments

if __name__ == "__main__":
    command_name, arguments = _parse_args()
    commands = {
        "register": register,
        "ping": ping,
        "poll": poll
    }
    try:
        commands[command_name](**arguments)
    except DataservClientException as e:
        print(e)
<commit_msg>fixed poll url<commit_after>#!/usr/bin/env python3

import sys
import time
import urllib
import urllib.error
import urllib.request
import argparse
import datetime
from datetime import timedelta

DEFAULT_URL = "http://104.236.104.117"
DEFAULT_DELAY = 15

_now = datetime.datetime.now

class DataservClientException(Exception):
    pass

class AddressAlreadyRegistered(DataservClientException):

    def __init__(self, address, url):
        msg = "Address {0} already registered at {1}!".format(address, url)
        super(AddressAlreadyRegistered, self).__init__(msg)

class FarmerNotFound(DataservClientException):

    def __init__(self, url):
        msg = "Farmer not found at {0}!".format(url)
        super(FarmerNotFound, self).__init__(msg)

class FarmerError(DataservClientException):

    def __init__(self, url):
        msg = "Farmer error at {0}!".format(url)
        super(FarmerError, self).__init__(msg)

class InvalidAddress(DataservClientException):

    def __init__(self, address):
        msg = "Address {0} not valid!".format(address)
        super(InvalidAddress, self).__init__(msg)

class ConnectionError(DataservClientException):

    def __init__(self, url):
        msg = "Could not connect to server {0}!".format(url)
        super(ConnectionError, self).__init__(msg)

def register(address, url=DEFAULT_URL):
    """Attempt to register the config address."""

    try:
        api_call = "{0}/api/register/{1}".format(url, address)
        response = urllib.request.urlopen(api_call)
        if response.code == 200:
            print("Address {0} now registered on {1}.".format(address, url))
            return True

    except urllib.error.HTTPError as e:
        if e.code == 409:
            raise AddressAlreadyRegistered(address, url)
        elif e.code == 404:
            raise FarmerNotFound(url)
        elif e.code == 400:
            raise InvalidAddress(address)
        elif e.code == 500:
            raise FarmerError(url)
        else:
            raise e
    except urllib.error.URLError:
        raise ConnectionError(url)

def ping(address, url=DEFAULT_URL):
    """Attempt keep-alive with the server."""
    try:
        print("Pinging {0} with address {1}.".format(url, address))
        api_call = "{0}/api/ping/{1}".format(url, address)
        urllib.request.urlopen(api_call)
        return True

    except urllib.error.HTTPError as e:
        if e.code == 400:
            raise InvalidAddress(address)
        elif e.code == 404:
            raise FarmerNotFound(url)
        elif e.code == 500:
            raise FarmerError(url)
        else:
            raise e
    except urllib.error.URLError:
        raise ConnectionError(url)

def poll(address, register_address=False, url=DEFAULT_URL,
         delay=DEFAULT_DELAY, limit=None):
    """TODO doc string"""
    stop_time = _now() + datetime.timedelta(seconds=limit) if limit else None

    if(register_address):
        register(address, url=url)

    while True:
        ping(address, url=url)
        if stop_time and _now() >= stop_time:
            return True
        time.sleep(delay)

def _parse_args():
    class ArgumentParser(argparse.ArgumentParser):
        def error(self, message):
            sys.stderr.write('error: %s\n' % message)
            self.print_help()
            sys.exit(2)

    # setup parser
    description = "Dataserve client command-line interface."
    parser = ArgumentParser(description=description)
    command_parser = parser.add_subparsers(
        title='commands', dest='command', metavar="<command>"
    )

    # register command
    register_parser = command_parser.add_parser(
        "register",
        help="Register a bitcoin address with farmer."
    )
    register_parser.add_argument(
        "address",
        help="Required bitcoin address."
    )
    register_parser.add_argument(
        "--url", default=DEFAULT_URL,
        help="Url of the farmer (default: {0}).".format(DEFAULT_URL)
    )

    # ping command
    ping_parser = command_parser.add_parser(
        "ping",
        help="Ping farmer with given address."
    )
    ping_parser.add_argument(
        "address",
        help="Required bitcoin address."
    )
    ping_parser.add_argument(
        "--url", default=DEFAULT_URL,
        help="Url of the farmer (default: {0}).".format(DEFAULT_URL)
    )

    # poll command
    poll_parser = command_parser.add_parser(
        "poll",
        help="Continuously ping farmer with given address."
    )
    poll_parser.add_argument(
        "address",
        help="Required bitcoin address."
    )
    poll_parser.add_argument(
        "--url", default=DEFAULT_URL,
        help="Url of the farmer (default: {0}).".format(DEFAULT_URL)
    )
    poll_parser.add_argument(
        "--delay", default=DEFAULT_DELAY,
        help="Deley between each ping."
    )
    poll_parser.add_argument(
        '--register', action='store_true',
        help="Register address before polling."
    )

    # get values
    arguments = vars(parser.parse_args())
    command_name = arguments.pop("command")
    if not command_name:
        parser.error("No command given!")
    return command_name, arguments

if __name__ == "__main__":
    command_name, arguments = _parse_args()
    commands = {
        "register": register,
        "ping": ping,
        "poll": poll
    }
    try:
        commands[command_name](**arguments)
    except DataservClientException as e:
        print(e)
<|endoftext|>


<commit_before>import aiohttp

import json
import logging
import re

import os
from typing import Text, List, Dict, Any, Union, Optional

from rasa.core import constants
from rasa.core.constants import INTENT_MESSAGE_PREFIX
from rasa.utils.endpoints import EndpointConfig

logger = logging.getLogger(__name__)

class NaturalLanguageInterpreter(object):
    async def parse(
        self, text: Text, message_id: Optional[Text] = None
    ) -> Dict[Text, Any]:
        raise NotImplementedError(
            "Interpreter needs to be able to parse messages into structured output."
        )

    @staticmethod
    def create(
        obj: Union[Text, "NaturalLanguageInterpreter"],
        endpoint: Optional[EndpointConfig] = None,
    ) -> "NaturalLanguageInterpreter":

        if isinstance(obj, NaturalLanguageInterpreter):
            return obj

        if not isinstance(obj, str):
            if obj is not None:
                logger.warning(
                    "Tried to create NLU interpreter "
                    "from '{}', which is not possible."
                    "Using RegexInterpreter instead."
                    "".format(obj)
                )
            return RegexInterpreter()

        if endpoint is None:
            if not os.path.exists(obj):
                logger.warning(
                    "No local NLU model '{}' found. Using RegexInterpreter instead.".format(
                        obj
                    )
                )
                return RegexInterpreter()
            else:
                return RasaNLUInterpreter(model_directory=obj)

        return RasaNLUHttpInterpreter(endpoint)

class RegexInterpreter(NaturalLanguageInterpreter):
    @staticmethod
    def allowed_prefixes():
        return INTENT_MESSAGE_PREFIX

    @staticmethod
    def _create_entities(parsed_entities: Dict[Text, Union[Text, List[Text]]], sidx: int, eidx: int) -> List[Dict[Text, Any]]:
        entities = []
        for k, vs in parsed_entities.items():
            if not isinstance(vs, list):
                vs = [vs]
            for value in vs:
                entities.append(
                    {
                        "entity": k,
                        "start": sidx,
                        "end": eidx,  # can't be more specific
                        "value": value,
                    }
                )
        return entities

    @staticmethod
    def _parse_parameters(
        entity_str: Text, sidx: int, eidx: int, user_input: Text
    ) -> List[Dict[Text, Any]]:
        if entity_str is None or not entity_str.strip():
            # if there is nothing to parse we will directly exit
            return []

        try:
            parsed_entities = json.loads(entity_str)
            if isinstance(parsed_entities, dict):
                return RegexInterpreter._create_entities(parsed_entities, sidx, eidx)
            else:
                raise Exception(
                    "Parsed value isn't a json object "
                    "(instead parser found '{}')"
                    ".".format(type(parsed_entities))
                )
        except Exception as e:
            logger.warning(
                "Invalid to parse arguments in line "
                "'{}'. Failed to decode parameters "
                "as a json object. Make sure the intent "
                "is followed by a proper json object. "
                "Error: {}".format(user_input, e)
            )
            return []

    @staticmethod
    def _parse_confidence(confidence_str: Text) -> float:
        if confidence_str is None:
            return 1.0

        try:
            return float(confidence_str.strip()[1:])
        except Exception as e:
            logger.warning(
                "Invalid to parse confidence value in line "
                "'{}'. Make sure the intent confidence is an "
                "@ followed by a decimal number. "
                "Error: {}".format(confidence_str, e)
            )
            return 0.0

    def _starts_with_intent_prefix(self, text: Text) -> bool:
        for c in self.allowed_prefixes():
            if text.startswith(c):
                return True
        return False

    @staticmethod
    def extract_intent_and_entities(user_input: Text) -> object:
        """Parse the user input using regexes to extract intent & entities."""

        prefixes = re.escape(RegexInterpreter.allowed_prefixes())
        # the regex matches "slot{"a": 1}"
        m = re.search("^[" + prefixes + "]?([^{@]+)(@[0-9.]+)?([{].+)?", user_input)
        if m is not None:
            event_name = m.group(1).strip()
            confidence = RegexInterpreter._parse_confidence(m.group(2))
            entities = RegexInterpreter._parse_parameters(
                m.group(3), m.start(3), m.end(3), user_input
            )

            return event_name, confidence, entities
        else:
            logger.warning(
                "Failed to parse intent end entities from '{}'. ".format(user_input)
            )
            return None, 0.0, []

    async def parse(
        self, text: Text, message_id: Optional[Text] = None
    ) -> Dict[Text, Any]:
        """Parse a text message."""

        intent, confidence, entities = self.extract_intent_and_entities(text)

        if self._starts_with_intent_prefix(text):
            message_text = text
        else:
            message_text = INTENT_MESSAGE_PREFIX + text

        return {
            "text": message_text,
            "intent": {"name": intent, "confidence": confidence},
            "intent_ranking": [{"name": intent, "confidence": confidence}],
            "entities": entities,
        }

class RasaNLUHttpInterpreter(NaturalLanguageInterpreter):
    def __init__(self, endpoint: EndpointConfig = None) -> None:
        if endpoint:
            self.endpoint = endpoint
        else:
            self.endpoint = EndpointConfig(constants.DEFAULT_SERVER_URL)

    async def parse(
        self, text: Text, message_id: Optional[int] = None
    ) -> Dict[Text, Any]:
        """Parse a text message.

        Return a default value if the parsing of the text failed."""

        default_return = {
            "intent": {"name": "", "confidence": 0.0},
            "entities": [],
            "text": "",
        }
        result = await self._rasa_http_parse(text)

        return result if result is not None else default_return

    async def _rasa_http_parse(self, text: Text) -> Optional[Dict[Text, Any]]:
        """Send a text message to a running rasa NLU http server.
        Return `None` on failure."""
        from requests.compat import urljoin

        if not self.endpoint:
            logger.error(
                "Failed to parse text '{}' using rasa NLU over http. "
                "No rasa NLU server specified!".format(text)
            )
            return None

        params = {"token": self.endpoint.token, "text": text}

        url = urljoin(self.endpoint.url, "/model/parse")

        # noinspection PyBroadException
        try:
            async with aiohttp.ClientSession() as session:
                async with session.post(url, json=params) as resp:
                    if resp.status == 200:
                        return await resp.json()
                    else:
                        logger.error(
                            "Failed to parse text '{}' using rasa NLU over "
                            "http. Error: {}".format(text, await resp.text())
                        )
                        return None
        except Exception:
            logger.exception(
                "Failed to parse text '{}' using rasa NLU over http.".format(text)
            )
            return None

class RasaNLUInterpreter(NaturalLanguageInterpreter):
    def __init__(
        self,
        model_directory: Text,
        config_file: Optional[Text] = None,
        lazy_init: bool = False,
    ):
        self.model_directory = model_directory
        self.lazy_init = lazy_init
        self.config_file = config_file

        if not lazy_init:
            self._load_interpreter()
        else:
            self.interpreter = None

    async def parse(
        self, text: Text, message_id: Optional[Text] = None
    ) -> Dict[Text, Any]:
        """Parse a text message.

        Return a default value if the parsing of the text failed."""

        if self.lazy_init and self.interpreter is None:
            self._load_interpreter()
        result = self.interpreter.parse(text)

        return result

    def _load_interpreter(self):
        from rasa.nlu.model import Interpreter

        self.interpreter = Interpreter.load(self.model_directory)
<commit_msg>Formatting.<commit_after>import aiohttp

import json
import logging
import re

import os
from typing import Text, List, Dict, Any, Union, Optional

from rasa.core import constants
from rasa.core.constants import INTENT_MESSAGE_PREFIX
from rasa.utils.endpoints import EndpointConfig

logger = logging.getLogger(__name__)

class NaturalLanguageInterpreter(object):
    async def parse(
        self, text: Text, message_id: Optional[Text] = None
    ) -> Dict[Text, Any]:
        raise NotImplementedError(
            "Interpreter needs to be able to parse messages into structured output."
        )

    @staticmethod
    def create(
        obj: Union[Text, "NaturalLanguageInterpreter"],
        endpoint: Optional[EndpointConfig] = None,
    ) -> "NaturalLanguageInterpreter":

        if isinstance(obj, NaturalLanguageInterpreter):
            return obj

        if not isinstance(obj, str):
            if obj is not None:
                logger.warning(
                    "Tried to create NLU interpreter "
                    "from '{}', which is not possible."
                    "Using RegexInterpreter instead."
                    "".format(obj)
                )
            return RegexInterpreter()

        if endpoint is None:
            if not os.path.exists(obj):
                logger.warning(
                    "No local NLU model '{}' found. Using RegexInterpreter instead.".format(
                        obj
                    )
                )
                return RegexInterpreter()
            else:
                return RasaNLUInterpreter(model_directory=obj)

        return RasaNLUHttpInterpreter(endpoint)

class RegexInterpreter(NaturalLanguageInterpreter):
    @staticmethod
    def allowed_prefixes():
        return INTENT_MESSAGE_PREFIX

    @staticmethod
    def _create_entities(
        parsed_entities: Dict[Text, Union[Text, List[Text]]], sidx: int, eidx: int
    ) -> List[Dict[Text, Any]]:
        entities = []
        for k, vs in parsed_entities.items():
            if not isinstance(vs, list):
                vs = [vs]
            for value in vs:
                entities.append(
                    {
                        "entity": k,
                        "start": sidx,
                        "end": eidx,  # can't be more specific
                        "value": value,
                    }
                )
        return entities

    @staticmethod
    def _parse_parameters(
        entity_str: Text, sidx: int, eidx: int, user_input: Text
    ) -> List[Dict[Text, Any]]:
        if entity_str is None or not entity_str.strip():
            # if there is nothing to parse we will directly exit
            return []

        try:
            parsed_entities = json.loads(entity_str)
            if isinstance(parsed_entities, dict):
                return RegexInterpreter._create_entities(parsed_entities, sidx, eidx)
            else:
                raise Exception(
                    "Parsed value isn't a json object "
                    "(instead parser found '{}')"
                    ".".format(type(parsed_entities))
                )
        except Exception as e:
            logger.warning(
                "Invalid to parse arguments in line "
                "'{}'. Failed to decode parameters "
                "as a json object. Make sure the intent "
                "is followed by a proper json object. "
                "Error: {}".format(user_input, e)
            )
            return []

    @staticmethod
    def _parse_confidence(confidence_str: Text) -> float:
        if confidence_str is None:
            return 1.0

        try:
            return float(confidence_str.strip()[1:])
        except Exception as e:
            logger.warning(
                "Invalid to parse confidence value in line "
                "'{}'. Make sure the intent confidence is an "
                "@ followed by a decimal number. "
                "Error: {}".format(confidence_str, e)
            )
            return 0.0

    def _starts_with_intent_prefix(self, text: Text) -> bool:
        for c in self.allowed_prefixes():
            if text.startswith(c):
                return True
        return False

    @staticmethod
    def extract_intent_and_entities(user_input: Text) -> object:
        """Parse the user input using regexes to extract intent & entities."""

        prefixes = re.escape(RegexInterpreter.allowed_prefixes())
        # the regex matches "slot{"a": 1}"
        m = re.search("^[" + prefixes + "]?([^{@]+)(@[0-9.]+)?([{].+)?", user_input)
        if m is not None:
            event_name = m.group(1).strip()
            confidence = RegexInterpreter._parse_confidence(m.group(2))
            entities = RegexInterpreter._parse_parameters(
                m.group(3), m.start(3), m.end(3), user_input
            )

            return event_name, confidence, entities
        else:
            logger.warning(
                "Failed to parse intent end entities from '{}'. ".format(user_input)
            )
            return None, 0.0, []

    async def parse(
        self, text: Text, message_id: Optional[Text] = None
    ) -> Dict[Text, Any]:
        """Parse a text message."""

        intent, confidence, entities = self.extract_intent_and_entities(text)

        if self._starts_with_intent_prefix(text):
            message_text = text
        else:
            message_text = INTENT_MESSAGE_PREFIX + text

        return {
            "text": message_text,
            "intent": {"name": intent, "confidence": confidence},
            "intent_ranking": [{"name": intent, "confidence": confidence}],
            "entities": entities,
        }

class RasaNLUHttpInterpreter(NaturalLanguageInterpreter):
    def __init__(self, endpoint: EndpointConfig = None) -> None:
        if endpoint:
            self.endpoint = endpoint
        else:
            self.endpoint = EndpointConfig(constants.DEFAULT_SERVER_URL)

    async def parse(
        self, text: Text, message_id: Optional[int] = None
    ) -> Dict[Text, Any]:
        """Parse a text message.

        Return a default value if the parsing of the text failed."""

        default_return = {
            "intent": {"name": "", "confidence": 0.0},
            "entities": [],
            "text": "",
        }
        result = await self._rasa_http_parse(text)

        return result if result is not None else default_return

    async def _rasa_http_parse(self, text: Text) -> Optional[Dict[Text, Any]]:
        """Send a text message to a running rasa NLU http server.
        Return `None` on failure."""
        from requests.compat import urljoin

        if not self.endpoint:
            logger.error(
                "Failed to parse text '{}' using rasa NLU over http. "
                "No rasa NLU server specified!".format(text)
            )
            return None

        params = {"token": self.endpoint.token, "text": text}

        url = urljoin(self.endpoint.url, "/model/parse")

        # noinspection PyBroadException
        try:
            async with aiohttp.ClientSession() as session:
                async with session.post(url, json=params) as resp:
                    if resp.status == 200:
                        return await resp.json()
                    else:
                        logger.error(
                            "Failed to parse text '{}' using rasa NLU over "
                            "http. Error: {}".format(text, await resp.text())
                        )
                        return None
        except Exception:
            logger.exception(
                "Failed to parse text '{}' using rasa NLU over http.".format(text)
            )
            return None

class RasaNLUInterpreter(NaturalLanguageInterpreter):
    def __init__(
        self,
        model_directory: Text,
        config_file: Optional[Text] = None,
        lazy_init: bool = False,
    ):
        self.model_directory = model_directory
        self.lazy_init = lazy_init
        self.config_file = config_file

        if not lazy_init:
            self._load_interpreter()
        else:
            self.interpreter = None

    async def parse(
        self, text: Text, message_id: Optional[Text] = None
    ) -> Dict[Text, Any]:
        """Parse a text message.

        Return a default value if the parsing of the text failed."""

        if self.lazy_init and self.interpreter is None:
            self._load_interpreter()
        result = self.interpreter.parse(text)

        return result

    def _load_interpreter(self):
        from rasa.nlu.model import Interpreter

        self.interpreter = Interpreter.load(self.model_directory)
<|endoftext|>


<commit_before>import cv2
import re
import logging
from os.path import join
from dataset.sample import Sample
from dataset.boundingbox import BoundingBox

logger = logging.getLogger(__name__)

def parse_image(line):
    filename = line.split(':')[-1].replace('"', '').strip()
    return cv2.imread(join('dataset', filename))

def parse_mask(line):
    filename = line.split(':')[-1].replace('"', '').strip()
    mask = cv2.imread(join('dataset', filename), cv2.CV_LOAD_IMAGE_GRAYSCALE)
    return cv2.threshold(mask, 0, 255, cv2.THRESH_BINARY)[1]

def parse_bbox(line):
    values = [int(s) - 1 for s in re.findall(r"[\w']+", line.split(':')[-1])]
    return BoundingBox.fromlist(values)

def parse(filename):
    """ parses the PASCAL compatible annotation file """
    if not filename:
        raise ValueError('filename can\'t be emtpy')

    sample = Sample()
    logger.info(filename)

    with open(filename) as f:
        for line in f:
            if sample.image is not None and line.startswith('Image filename'):
                sample.image = parse_image(line)
                continue

            if sample.mask is not None and line.startswith('Pixel mask'):
                sample.mask = parse_mask(line)
                continue

            if line.startswith('Bounding box'):
                sample.bboxes.append(parse_bbox(line))

    return sample
<commit_msg>added __doc__ for pascal parser<commit_after>import cv2
import re
import logging
from os.path import join
from dataset.sample import Sample
from dataset.boundingbox import BoundingBox

logger = logging.getLogger(__name__)

def parse_image(line):
    """ parses the string formated as the PASCAL image filename annotation

        e.g. Image filename : "PennFudanPed/PNGImages/FudanPed00001.png"

        line: string
            line in the PASCAL annotation file with image information

        returns: array
            an image array parsed from the given line
    """
    filename = line.split(':')[-1].replace('"', '').strip()
    return cv2.imread(join('dataset', filename))

def parse_mask(line):
    """ parses the string formated as the PASCAL pixel mask annotation

        e.g. Pixel mask for object 1 "PASpersonWalking" :
            "PennFudanPed/PedMasks/FudanPed00001_mask.png"

        line: string
            line in the PASCAL annotation file with mask information

        returns: array
            a mask array parsed from the given line
    """
    filename = line.split(':')[-1].replace('"', '').strip()
    mask = cv2.imread(join('dataset', filename), cv2.CV_LOAD_IMAGE_GRAYSCALE)
    return cv2.threshold(mask, 0, 255, cv2.THRESH_BINARY)[1]

def parse_bbox(line):
    """ parses the string formated as the PASCAL bounding box annotation

        e.g. Bounding box for object 2 "PASpersonWalking"
                (Xmin, Ymin) - (Xmax, Ymax) : (420, 171) - (535, 486)

        line: string
            line in the PASCAL annotation file with bounding box information

        returns: BoundingBox
            a bounding box parsed from the given line
    """
    values = [int(s) - 1 for s in re.findall(r"[\w']+", line.split(':')[-1])]
    return BoundingBox.fromlist(values)

def parse(filename):
    """ parses the PASCAL compatible annotation file

    filename: string
        path to the annotation file

    returns: Sample
        instance of the sample class parsed from th e annotation file
    """
    if not filename:
        raise ValueError('filename can\'t be emtpy')

    sample = Sample()
    logger.info(filename)

    with open(filename) as f:
        for line in f:
            if sample.image is not None and line.startswith('Image filename'):
                sample.image = parse_image(line)
                continue

            if sample.mask is not None and line.startswith('Pixel mask'):
                sample.mask = parse_mask(line)
                continue

            if line.startswith('Bounding box'):
                sample.bboxes.append(parse_bbox(line))

    return sample
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from django.contrib.auth.backends import ModelBackend
import logging

LOG = logging.getLogger(__name__)

class DbaasBackend(ModelBackend):

    def get_all_permissions(self, user_obj, obj=None):
        if user_obj.is_anonymous() or obj is not None:
            return set()
        if not hasattr(user_obj, '_perm_cache'):
            user_obj._perm_cache = set(["%s.%s" % (p.content_type.app_label, p.codename) for p in user_obj.user_permissions.select_related()])
            user_obj._perm_cache.update(self.get_group_permissions(user_obj))
        return user_obj._perm_cache

    def has_perm(self, user_obj, perm, obj=None):
        #LOG.debug("validating perm %s for user %s on object %s" % (perm, user_obj, obj))
        # We are using django_services in admin, but it does not provide any support to
        # object level permission
        if not user_obj.is_active:
            return False
        else:
            return perm in user_obj.get_all_permissions(obj=None)
<commit_msg>get team permissions<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from django.contrib.auth.backends import ModelBackend
import logging

from account.models import Team

LOG = logging.getLogger(__name__)

class DbaasBackend(ModelBackend):

    def get_all_permissions(self, user_obj, obj=None):
        print "get_all_permissions for user: %s" % user_obj
        if user_obj.is_anonymous() or obj is not None:
            return set()
        if not hasattr(user_obj, '_perm_cache'):
            user_obj._perm_cache = set(["%s.%s" % (p.content_type.app_label, p.codename) for p in user_obj.user_permissions.select_related()])
            user_obj._perm_cache.update(self.get_group_permissions(user_obj))
            #get team permissions
            user_obj._perm_cache.update(Team.get_all_permissions_for(user=user_obj))

        return user_obj._perm_cache

    def has_perm(self, user_obj, perm, obj=None):
        LOG.debug("validating perm %s for user %s on object %s" % (perm, user_obj, obj))
        # We are using django_services in admin, but it does not provide any support to
        # object level permission
        if not user_obj.is_active:
            return False
        else:
            return perm in user_obj.get_all_permissions(obj=None)
<|endoftext|>