<commit_before>#!/usr/bin/env python3
#
# Copyright 2016 Red Hat, Inc.
#
# Authors:
#     Fam Zheng <famz@redhat.com>
#
# This work is licensed under the MIT License.  Please see the LICENSE file or
# http://opensource.org/licenses/MIT.

from .models import Message, MessageProperty, MessageResult, Result, QueuedSeries
from functools import reduce

from django.db import connection
from django.db.models import Q

from django.contrib.postgres.search import SearchQuery, SearchVector, SearchVectorField
from django.db.models import Lookup
from django.db.models.fields import Field

@Field.register_lookup
class NotEqual(Lookup):
    lookup_name = 'ne'

    def as_sql(self, compiler, connection):
        lhs, lhs_params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.process_rhs(compiler, connection)
        params = lhs_params + rhs_params
        return '%s <> %s' % (lhs, rhs), params

class InvalidSearchTerm(Exception):
    pass

# Hack alert: Django wraps each argument to to_tsvector with a COALESCE function,
# and that causes postgres not to use the index.  Monkeypatch the constructor
# to skip that step, which we do not need since the subject field is not nullable.
class NonNullSearchVector(SearchVector):
    function = 'to_tsvector'
    arg_joiner = " || ' ' || "
    _output_field = SearchVectorField()
    config = None

    def __init__(self, *expressions, **extra):
        super(SearchVector, self).__init__(*expressions, **extra)
        self.config = self.extra.get('config', self.config)
        self.weight = None

class SearchEngine(object):
    """

The general form of search string is a list of terms separated with space:

    QUERY = TERM TERM ...

Each term can be either a plain keyword, or a predict in the form of
`PRED:EXP`, where PRED is the predefined filter and EXP is the parameters to be
applied to the filter. As a simple example:

    bugfix from:Bob to:George age:>1w

to search emails titled as 'bugfix' (a subject keyword filter) from Bob (a
sender filter) to George (a recipient filter) before 1 week ago (an age
filter).

or

    bugfix from:Bob is:reviewed not:obsoleted

to search all emails from Bob that have "bugfix" in subject, and have been
reviewed but is not obsoleted (by a new revision of this series). Because there
are syntax shortcut for some predicts, it can be simplified as:

    from:Bob fix +reviewed -tested

---

## Supported filter types

### Search by age

 - Syntax: age:AGE
 - Syntax: >AGE
 - Syntax: <AGE

Filter by age of the message. Supports "d" (day), "w" (week), "m" (month) and "y" (year) as units. Examples:

 - age:1d
 - age:>2d
 - age:<1w
 - <1m
 - \>1w

---

### Search by series state

Syntax:

 - is:reviewed - all the patches in the series is reviewed
 - is:obsolete or is:old - the series has newer version
 - is:complete - the series has all the patches it contains
 - is:merged - the series is included in the project's git tree
 - is:pull - the series is a pull request
 - is:applied - a git tree is available for the series
 - has:replies - the series received a reply (apart from patches sent by the submitter)

Example:

    is:reviewed

"not:X" is the opposite of "is:X". "+X" and "-X" are shorter synonyms of "is:X"
and "not:X" respectively.

---

### Search addresses

 - Syntax: from:ADDRESS
 - Syntax: to:ADDRESS

Compare the address info of message. Example:

    from:alice to:bob

---

### Search by result

Syntax:

 - pending:NAME, failure:NAME, running:NAME - any result with the given
   name is in the pending/failure/running state
 - success:NAME - all results with the given name are in the success state
   (and there is at least one result with the given name)

where NAME can be e.g. "git", "testing", "testing.TEST-NAME"

Example:

    success:git
    failure:testing.FreeBSD

---

### Search by review state

Syntax:

 - accept:USERNAME or ack:USERNAME - the series was marked as accepted by the user
 - reject:USERNAME or nack:USERNAME - the series was marked as reject by the user
 - review:USERNAME - the series was marked as accepted or rejected by the user

USERNAME can be "me" to identify the current user

---

### Reverse condition

 - Syntax: !TERM

Negative of an expression. Example:

    !is:reviewed     (query series that are not reviewed)
    !has:replies     (query series that have not received any comment)

---

### Search by message id

 - Syntax: id:MESSAGE-ID

Exact match of message-id. Example:

    id:<1416902879-17422-1-git-send-email-user@domain.com>

or

    id:1416902879-17422-1-git-send-email-user@domain.com

---

### Search by text

 - Syntax: KEYWORD

Search text keyword in the email message. Example:

    regression

"""
    def _make_filter_subquery(self, model, q):
        message_ids = model.objects.filter(q).values('message_id')
        return Q(id__in=message_ids)

    def _make_filter_result(self, term, **kwargs):
        q = Q(name=term, **kwargs) | Q(name__startswith=term + '.', **kwargs)
        return self._make_filter_subquery(MessageResult, q)

    def _make_filter_age(self, cond):
        import datetime

        def human_to_seconds(n, unit):
            if unit == "d":
                return n * 86400
            elif unit == "w":
                return n * 86400 * 7
            elif unit == "m":
                return n * 86400 * 30
            elif unit == "y":
                return n * 86400 * 365
            raise Exception("No unit specified")

        if cond.startswith("<"):
            less = True
            cond = cond[1:]
        elif cond.startswith(">"):
            less = False
            cond = cond[1:]
        else:
            less = False
        num, unit = cond[:-1], cond[-1].lower()
        if not num.isdigit() or unit not in "dwmy":
            raise InvalidSearchTerm("Invalid age string: %s" % cond)
        sec = human_to_seconds(int(num), unit)
        p = datetime.datetime.now() - datetime.timedelta(0, sec)
        if less:
            q = Q(date__gte=p)
        else:
            q = Q(date__lte=p)
        return q

    def _add_to_keywords(self, t):
        self._last_keywords.append(t)
        return Q()

    def _make_filter_is(self, cond):
        if cond == "complete":
            return Q(is_complete=True)
        elif cond == "pull":
            self._add_to_keywords('PULL')
            return Q(subject__contains='[PULL') | Q(subject__contains='[GIT PULL')
        elif cond == "reviewed":
            return self._make_filter_subquery(MessageProperty, Q(name="reviewed", value=True))
        elif cond in ("obsoleted", "old"):
            return self._make_filter_subquery(
                MessageProperty,
                Q(name="obsoleted-by", value__isnull=False) & ~Q(name="obsoleted-by", value__iexact='')
            )
        elif cond == "applied":
            return self._make_filter_subquery(MessageResult, Q(name="git", status=Result.SUCCESS))
        elif cond == "tested":
            return self._make_filter_subquery(MessageProperty, Q(name="testing.done", value=True))
        elif cond == "merged":
            return Q(is_merged=True)
        return None

    def _make_filter_review(self, username, user, **kwargs):
        if username == "me":
            if not user.is_authenticated:
                # Django hack to return an always false Q object
                return Q(pk=None)
            q = Q(user=user, **kwargs)
        else:
            q = Q(user__username=username, **kwargs)
        return self._make_filter_subquery(QueuedSeries, q)

    def _make_filter(self, term, user):
        if term.startswith("age:"):
            cond = term[term.find(":") + 1:]
            return self._make_filter_age(cond)
        elif term[0] in "<>" and len(term) > 1:
            return self._make_filter_age(term)
        elif term.startswith("from:"):
            cond = term[term.find(":") + 1:]
            return Q(sender__icontains=cond)
        elif term.startswith("to:"):
            cond = term[term.find(":") + 1:]
            return Q(recipients__icontains=cond)
        elif term.startswith("subject:"):
            cond = term[term.find(":") + 1:]
            return self._add_to_keywords(cond)
        elif term.startswith("id:"):
            cond = term[term.find(":") + 1:]
            if cond[0] == "<" and cond[-1] == ">":
                cond = cond[1:-1]
            return Q(message_id=cond)
        elif term.startswith("is:"):
            return self._make_filter_is(term[3:]) or self._add_to_keywords(term)
        elif term.startswith("not:"):
            return ~self._make_filter_is(term[4:]) or self._add_to_keywords(term)
        elif term.startswith("has:"):
            cond = term[term.find(":") + 1:]
            if cond == "replies":
                return Q(last_comment_date__isnull=False)
            else:
                return Q(properties__name=cond)
        elif term.startswith("failure:"):
            return self._make_filter_result(term[8:], status=Result.FAILURE)
        elif term.startswith("success:"):
            # What we want is "all results are successes", but the only way to
            # express it is "there is a result and not (any result is not a success)".
            return self._make_filter_result(term[8:]) \
                & ~self._make_filter_result(term[8:], status__ne=Result.SUCCESS)
        elif term.startswith("pending:"):
            return self._make_filter_result(term[8:], status=Result.PENDING)
        elif term.startswith("running:"):
            return self._make_filter_result(term[8:], status=Result.RUNNING)
        elif term.startswith("ack:") or term.startswith("accept:") or term.startswith("accepted:"):
            username = term[term.find(":") + 1:]
            return self._make_filter_review(username, user, name="accept")
        elif term.startswith("nack:") or term.startswith("reject:") or term.startswith("rejected:"):
            username = term[term.find(":") + 1:]
            return self._make_filter_review(username, user, name="reject")
        elif term.startswith("review:") or term.startswith("reviewed:"):
            username = term[term.find(":") + 1:]
            return self._make_filter_review(username, user, name="accept")
        elif term.startswith("project:"):
            cond = term[term.find(":") + 1:]
            self._projects.add(cond)
            return Q(project__name=cond) | Q(project__parent_project__name=cond)

        # Keyword in subject is the default
        return self._add_to_keywords(term)

    def _process_term(self, term, user):
        """ Return a Q object that will be applied to the query """
        is_plusminus = neg = False
        if term[0] in "+-!":
            neg = (term[0] != "+")
            is_plusminus = (term[0] != "!")
            term = term[1:]

        if is_plusminus and ":" not in term:
            q = self._make_filter_is(term) or self._add_to_keywords(term)
        else:
            q = self._make_filter(term, user)
        if neg:
            return ~q
        else:
            return q

    def last_keywords(self):
        return getattr(self, "_last_keywords", [])

    def project(self):
        return next(iter(self._projects)) if len(self._projects) == 1 else None

    def search_series(self, *terms, user=None, queryset=None):
        self._last_keywords = []
        self._projects = set()
        q = reduce(
            lambda x, y: x & y,
            map(lambda t: self._process_term(t, user), terms),
            Q()
        )
        if queryset is None:
            queryset = Message.objects.series_heads()
        if self._last_keywords:
            if connection.vendor == 'postgresql':
                queryset = queryset.annotate(subjsearch=NonNullSearchVector('subject', config='english'))
                searchq = reduce(lambda x, y: x & y,
                                 map(lambda x: SearchQuery(x, config='english'), self._last_keywords))
                q = q & Q(subjsearch=searchq)
            else:
                q = reduce(lambda x, y: x & Q(subject__icontains=y),
                           self._last_keywords, q)

        return queryset.filter(q)

    def query_test_message(self, query, message):
        queryset = Message.objects.filter(id=message.id)
        terms = [x.strip() for x in query.split() if x.strip()]
        return self.search_series(*terms, queryset=queryset).first()

<commit_msg>search: Introduce maintained-by:NAME term<commit_after>#!/usr/bin/env python3
#
# Copyright 2016 Red Hat, Inc.
#
# Authors:
#     Fam Zheng <famz@redhat.com>
#
# This work is licensed under the MIT License.  Please see the LICENSE file or
# http://opensource.org/licenses/MIT.

from .models import Message, MessageProperty, MessageResult, Result, QueuedSeries
from functools import reduce

from django.db import connection
from django.db.models import Q

from django.contrib.postgres.search import SearchQuery, SearchVector, SearchVectorField
from django.db.models import Lookup
from django.db.models.fields import Field

@Field.register_lookup
class NotEqual(Lookup):
    lookup_name = 'ne'

    def as_sql(self, compiler, connection):
        lhs, lhs_params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.process_rhs(compiler, connection)
        params = lhs_params + rhs_params
        return '%s <> %s' % (lhs, rhs), params

class InvalidSearchTerm(Exception):
    pass

# Hack alert: Django wraps each argument to to_tsvector with a COALESCE function,
# and that causes postgres not to use the index.  Monkeypatch the constructor
# to skip that step, which we do not need since the subject field is not nullable.
class NonNullSearchVector(SearchVector):
    function = 'to_tsvector'
    arg_joiner = " || ' ' || "
    _output_field = SearchVectorField()
    config = None

    def __init__(self, *expressions, **extra):
        super(SearchVector, self).__init__(*expressions, **extra)
        self.config = self.extra.get('config', self.config)
        self.weight = None

class SearchEngine(object):
    """

The general form of search string is a list of terms separated with space:

    QUERY = TERM TERM ...

Each term can be either a plain keyword, or a predict in the form of
`PRED:EXP`, where PRED is the predefined filter and EXP is the parameters to be
applied to the filter. As a simple example:

    bugfix from:Bob to:George age:>1w

to search emails titled as 'bugfix' (a subject keyword filter) from Bob (a
sender filter) to George (a recipient filter) before 1 week ago (an age
filter).

or

    bugfix from:Bob is:reviewed not:obsoleted

to search all emails from Bob that have "bugfix" in subject, and have been
reviewed but is not obsoleted (by a new revision of this series). Because there
are syntax shortcut for some predicts, it can be simplified as:

    from:Bob fix +reviewed -tested

---

## Supported filter types

### Search by age

 - Syntax: age:AGE
 - Syntax: >AGE
 - Syntax: <AGE

Filter by age of the message. Supports "d" (day), "w" (week), "m" (month) and "y" (year) as units. Examples:

 - age:1d
 - age:>2d
 - age:<1w
 - <1m
 - \>1w

---

### Search by series state

Syntax:

 - is:reviewed - all the patches in the series is reviewed
 - is:obsolete or is:old - the series has newer version
 - is:complete - the series has all the patches it contains
 - is:merged - the series is included in the project's git tree
 - is:pull - the series is a pull request
 - is:applied - a git tree is available for the series
 - has:replies - the series received a reply (apart from patches sent by the submitter)

Example:

    is:reviewed

"not:X" is the opposite of "is:X". "+X" and "-X" are shorter synonyms of "is:X"
and "not:X" respectively.

---

### Search addresses

 - Syntax: from:ADDRESS
 - Syntax: to:ADDRESS

Compare the address info of message. Example:

    from:alice to:bob

---

### Search by maintainer associated with the changeset

 - Syntax: maintained-by:NAME
 - Syntax: maint:NAME

NAME can be the name, email or a substring of MAINTAINERS file entries of the
maintainer.

---

### Search by result

Syntax:

 - pending:NAME, failure:NAME, running:NAME - any result with the given
   name is in the pending/failure/running state
 - success:NAME - all results with the given name are in the success state
   (and there is at least one result with the given name)

where NAME can be e.g. "git", "testing", "testing.TEST-NAME"

Example:

    success:git
    failure:testing.FreeBSD

---

### Search by review state

Syntax:

 - accept:USERNAME or ack:USERNAME - the series was marked as accepted by the user
 - reject:USERNAME or nack:USERNAME - the series was marked as reject by the user
 - review:USERNAME - the series was marked as accepted or rejected by the user

USERNAME can be "me" to identify the current user

---

### Reverse condition

 - Syntax: !TERM

Negative of an expression. Example:

    !is:reviewed     (query series that are not reviewed)
    !has:replies     (query series that have not received any comment)

---

### Search by message id

 - Syntax: id:MESSAGE-ID

Exact match of message-id. Example:

    id:<1416902879-17422-1-git-send-email-user@domain.com>

or

    id:1416902879-17422-1-git-send-email-user@domain.com

---

### Search by text

 - Syntax: KEYWORD

Search text keyword in the email message. Example:

    regression

"""
    def _make_filter_subquery(self, model, q):
        message_ids = model.objects.filter(q).values('message_id')
        return Q(id__in=message_ids)

    def _make_filter_result(self, term, **kwargs):
        q = Q(name=term, **kwargs) | Q(name__startswith=term + '.', **kwargs)
        return self._make_filter_subquery(MessageResult, q)

    def _make_filter_age(self, cond):
        import datetime

        def human_to_seconds(n, unit):
            if unit == "d":
                return n * 86400
            elif unit == "w":
                return n * 86400 * 7
            elif unit == "m":
                return n * 86400 * 30
            elif unit == "y":
                return n * 86400 * 365
            raise Exception("No unit specified")

        if cond.startswith("<"):
            less = True
            cond = cond[1:]
        elif cond.startswith(">"):
            less = False
            cond = cond[1:]
        else:
            less = False
        num, unit = cond[:-1], cond[-1].lower()
        if not num.isdigit() or unit not in "dwmy":
            raise InvalidSearchTerm("Invalid age string: %s" % cond)
        sec = human_to_seconds(int(num), unit)
        p = datetime.datetime.now() - datetime.timedelta(0, sec)
        if less:
            q = Q(date__gte=p)
        else:
            q = Q(date__lte=p)
        return q

    def _add_to_keywords(self, t):
        self._last_keywords.append(t)
        return Q()

    def _make_filter_is(self, cond):
        if cond == "complete":
            return Q(is_complete=True)
        elif cond == "pull":
            self._add_to_keywords('PULL')
            return Q(subject__contains='[PULL') | Q(subject__contains='[GIT PULL')
        elif cond == "reviewed":
            return self._make_filter_subquery(MessageProperty, Q(name="reviewed", value=True))
        elif cond in ("obsoleted", "old"):
            return self._make_filter_subquery(
                MessageProperty,
                Q(name="obsoleted-by", value__isnull=False) & ~Q(name="obsoleted-by", value__iexact='')
            )
        elif cond == "applied":
            return self._make_filter_subquery(MessageResult, Q(name="git", status=Result.SUCCESS))
        elif cond == "tested":
            return self._make_filter_subquery(MessageProperty, Q(name="testing.done", value=True))
        elif cond == "merged":
            return Q(is_merged=True)
        return None

    def _make_filter_review(self, username, user, **kwargs):
        if username == "me":
            if not user.is_authenticated:
                # Django hack to return an always false Q object
                return Q(pk=None)
            q = Q(user=user, **kwargs)
        else:
            q = Q(user__username=username, **kwargs)
        return self._make_filter_subquery(QueuedSeries, q)

    def _make_filter(self, term, user):
        if term.startswith("age:"):
            cond = term[term.find(":") + 1:]
            return self._make_filter_age(cond)
        elif term[0] in "<>" and len(term) > 1:
            return self._make_filter_age(term)
        elif term.startswith("from:"):
            cond = term[term.find(":") + 1:]
            return Q(sender__icontains=cond)
        elif term.startswith("to:"):
            cond = term[term.find(":") + 1:]
            return Q(recipients__icontains=cond)
        elif term.startswith("subject:"):
            cond = term[term.find(":") + 1:]
            return self._add_to_keywords(cond)
        elif term.startswith("id:"):
            cond = term[term.find(":") + 1:]
            if cond[0] == "<" and cond[-1] == ">":
                cond = cond[1:-1]
            return Q(message_id=cond)
        elif term.startswith("is:"):
            return self._make_filter_is(term[3:]) or self._add_to_keywords(term)
        elif term.startswith("not:"):
            return ~self._make_filter_is(term[4:]) or self._add_to_keywords(term)
        elif term.startswith("has:"):
            cond = term[term.find(":") + 1:]
            if cond == "replies":
                return Q(last_comment_date__isnull=False)
            else:
                return Q(properties__name=cond)
        elif term.startswith("failure:"):
            return self._make_filter_result(term[8:], status=Result.FAILURE)
        elif term.startswith("success:"):
            # What we want is "all results are successes", but the only way to
            # express it is "there is a result and not (any result is not a success)".
            return self._make_filter_result(term[8:]) \
                & ~self._make_filter_result(term[8:], status__ne=Result.SUCCESS)
        elif term.startswith("pending:"):
            return self._make_filter_result(term[8:], status=Result.PENDING)
        elif term.startswith("running:"):
            return self._make_filter_result(term[8:], status=Result.RUNNING)
        elif term.startswith("ack:") or term.startswith("accept:") or term.startswith("accepted:"):
            username = term[term.find(":") + 1:]
            return self._make_filter_review(username, user, name="accept")
        elif term.startswith("nack:") or term.startswith("reject:") or term.startswith("rejected:"):
            username = term[term.find(":") + 1:]
            return self._make_filter_review(username, user, name="reject")
        elif term.startswith("review:") or term.startswith("reviewed:"):
            username = term[term.find(":") + 1:]
            return self._make_filter_review(username, user, name="accept")
        elif term.startswith("project:"):
            cond = term[term.find(":") + 1:]
            self._projects.add(cond)
            return Q(project__name=cond) | Q(project__parent_project__name=cond)
        elif term.startswith("maintained-by:") or term.startswith("maint:"):
            cond = term[term.find(":") + 1:]
            if cond == "me" and user:
                cond = user.email
            return Q(maintainers__icontains=cond)

        # Keyword in subject is the default
        return self._add_to_keywords(term)

    def _process_term(self, term, user):
        """ Return a Q object that will be applied to the query """
        is_plusminus = neg = False
        if term[0] in "+-!":
            neg = (term[0] != "+")
            is_plusminus = (term[0] != "!")
            term = term[1:]

        if is_plusminus and ":" not in term:
            q = self._make_filter_is(term) or self._add_to_keywords(term)
        else:
            q = self._make_filter(term, user)
        if neg:
            return ~q
        else:
            return q

    def last_keywords(self):
        return getattr(self, "_last_keywords", [])

    def project(self):
        return next(iter(self._projects)) if len(self._projects) == 1 else None

    def search_series(self, *terms, user=None, queryset=None):
        self._last_keywords = []
        self._projects = set()
        q = reduce(
            lambda x, y: x & y,
            map(lambda t: self._process_term(t, user), terms),
            Q()
        )
        if queryset is None:
            queryset = Message.objects.series_heads()
        if self._last_keywords:
            if connection.vendor == 'postgresql':
                queryset = queryset.annotate(subjsearch=NonNullSearchVector('subject', config='english'))
                searchq = reduce(lambda x, y: x & y,
                                 map(lambda x: SearchQuery(x, config='english'), self._last_keywords))
                q = q & Q(subjsearch=searchq)
            else:
                q = reduce(lambda x, y: x & Q(subject__icontains=y),
                           self._last_keywords, q)

        return queryset.filter(q)

    def query_test_message(self, query, message):
        queryset = Message.objects.filter(id=message.id)
        terms = [x.strip() for x in query.split() if x.strip()]
        return self.search_series(*terms, queryset=queryset).first()

<|endoftext|>


<commit_before>import dpath.path

def search(obj, glob, yielded=False):
    """
    Given a path glob, return a dictionary containing all keys
    that matched the given glob.

    If 'yielded' is true, then a dictionary will not be returned.
    Instead tuples will be yielded in the form of (path, value) for
    every element in the document that matched the glob. The path
    will be a list of separated path elements constructing the path.
    ("/".join() them again if that's what you want.)
    """
    if yielded:
        return _search_yielded(obj, glob)
    return _search_view(obj, glob)
def _search_view(obj, glob):
    view = {}
    for path in dpath.path.search(obj, glob.split("/")):
        dpath.path.merge(view, dpath.path.get(obj, path, view=True))
    return view

def _search_yielded(obj, glob):
    for path in dpath.path.search(obj, glob.split("/")):
        yield (path, dpath.path.get(obj, path))

def merge(dst, src):
    """Merge source into destination. Like dict.copy() but performs
    deep merging."""
    if isinstance(src, dict):
        for (i, v) in enumerate(src):
            if not v in dst:
                dst[v] = src[v]
            else:
                if not isinstance(src[v], (dict, list)):
                    dst[v] = src[v]
                else:
                    merge(dst[v], src[v])
    elif isinstance(src, list):
        for (i, v) in enumerate(src):
            if i >= len(dst):
                dst += [None] * (i - len(dst) + 1)
            if dst[i] == None:
                dst[i] = src[i]
            else:
                if not isinstance(src[i], (dict, list)):
                    dst[i] = src[i]
                else:
                    merge(dst[i], src[i])

<commit_msg>Made _search_* functions inline<commit_after>import dpath.path

def search(obj, glob, yielded=False):
    """
    Given a path glob, return a dictionary containing all keys
    that matched the given glob.

    If 'yielded' is true, then a dictionary will not be returned.
    Instead tuples will be yielded in the form of (path, value) for
    every element in the document that matched the glob. The path
    will be a list of separated path elements constructing the path.
    ("/".join() them again if that's what you want.)
    """

    def _search_view(obj, glob):
        view = {}
        for path in dpath.path.search(obj, glob.split("/")):
            dpath.path.merge(view, dpath.path.get(obj, path, view=True))
        return view

    def _search_yielded(obj, glob):
        for path in dpath.path.search(obj, glob.split("/")):
            yield (path, dpath.path.get(obj, path))

    if yielded:
        return _search_yielded(obj, glob)
    return _search_view(obj, glob)

def merge(dst, src):
    """Merge source into destination. Like dict.copy() but performs
    deep merging."""
    if isinstance(src, dict):
        for (i, v) in enumerate(src):
            if not v in dst:
                dst[v] = src[v]
            else:
                if not isinstance(src[v], (dict, list)):
                    dst[v] = src[v]
                else:
                    merge(dst[v], src[v])
    elif isinstance(src, list):
        for (i, v) in enumerate(src):
            if i >= len(dst):
                dst += [None] * (i - len(dst) + 1)
            if dst[i] == None:
                dst[i] = src[i]
            else:
                if not isinstance(src[i], (dict, list)):
                    dst[i] = src[i]
                else:
                    merge(dst[i], src[i])

<|endoftext|>


<commit_before>try:
    import cmemcache as memcache
except:
    import memcache
from addpinyin import *
import hashlib
import datetime
import cjson

url = ''
title = ''
text = '''

'''
pinyin = addpinyin(text)
now = datetime.datetime.now()

key = "%s" % hashlib.md5(url).hexdigest()
kid = string.atoi(key[:10],16)

mc = memcache.Client(['114.113.30.29:11211'])
dbvalue=cjson.encode({"title":ttl,"url":url,"html":text,"text":text,"datetime":str(nowtime),"addpinyin":pinyin,"body":text,"kid":kid})
mc.set(str(kid),dbvalue)

cmd = 'tctmgr put infodb/infodb %s "title" "%s" "savedate" "%s" "tag1" "%s"' % (kid,title,now.date(),'专家咨询')

<commit_msg>合并线上代码 加入log<commit_after># encoding: utf-8
try:
    import cmemcache as memcache
except:
    import memcache
from addpinyin import *
import hashlib
import datetime
import cjson
import string

url = ''
tag = '专家咨询'
title = ''
text = '''

'''
pinyin = addpinyin(text)
now = datetime.datetime.now()

key = "%s" % hashlib.md5(url).hexdigest()
kid = string.atoi(key[:10],16)

print kid,title

mc = memcache.Client(['114.113.30.29:11211'])
dbvalue=cjson.encode({"title":title,"url":url,"html":text,"text":text,"datetime":str(nowtime),"addpinyin":pinyin,"body":text,"kid":kid})
mc.set(str(kid),dbvalue)

cmd = 'tctmgr put infodb/infodb %s "title" "%s" "savedate" "%s" "tag1" "%s"' % (kid,title,now.date(),tag)

<|endoftext|>


<commit_before>import json
import io
from gensim import corpora

# skip-gram 모델 만들기
# batch-set 만들기
# 임베딩테이블 만들기
# 학습하기

dic = dict()
rdic = []

with io.open('/Users/KYD/Dropbox/논문/MSCOCO/captions_train-val2014/annotations/word_to_index_dict.json', 'r') as f:
    dic = json.loads(f.read())

with io.open('/Users/KYD/Dropbox/논문/MSCOCO/captions_train-val2014/annotations/index_to_word_list.json', 'r') as f:
    rdic = json.loads(f.read())

<commit_msg>updating text2sequence.py v2<commit_after>import json
import io
from gensim import corpora

# skip-gram 모델 만들기
# batch-set 만들기
# 임베딩테이블 만들기
# 학습하기

dic = dict()
rdic = []
window_size = 2

with io.open('/Users/KYD/Dropbox/논문/MSCOCO/captions_train-val2014/annotations/word_to_index_dict.json', 'r') as f:
    dic = json.loads(f.read())

with io.open('/Users/KYD/Dropbox/논문/MSCOCO/captions_train-val2014/annotations/index_to_word_list.json', 'r') as f:
    rdic = json.loads(f.read())

index_data = []
with io.open('/Users/KYD/Dropbox/논문/MSCOCO/captions_train-val2014/annotations/trimmed_tokenized_caption.json', 'r') as f:
    trimmed_tokenized_caption = json.loads(f.read())

    for caption in trimmed_tokenized_caption:
        templist = []
        for token in caption:
            templist.append(dic[token])

        index_data.append(templist)

print(index_data[:10])
#skip-gram pair 만들기

skip_gram_pairs = []

for sequence in index_data:
    for target_index in range(len(sequence)):
        for window_index in range(0 - window_size, window_size+1):
            if 0 <= target_index + window_index < len(sequence) and target_index is not window_index + target_index:
                skip_gram_pairs.append([sequence[target_index], sequence[target_index + window_index]])

print(skip_gram_pairs[:10])<|endoftext|>


<commit_before>"""

Copyright 2019, Institute for Systems Biology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import logging
import base64
import json
import requests

from flask import request, jsonify

logger = logging.getLogger(__name__)

# BEGIN METHODS
def _base64_decode(encoded_str):
    # Add paddings manually if necessary.
    num_missed_paddings = 4 - len(encoded_str) % 4
    if num_missed_paddings != 4:
        encoded_str += b'=' * num_missed_paddings
    return base64.b64decode(encoded_str).decode('utf-8')

def auth_info():
    """Retrieves the authenication information from Google Cloud Endpoints."""
    encoded_info = request.headers.get('X-Endpoint-API-UserInfo', None)

    if encoded_info:
        info_json = _base64_decode(encoded_info)
        user_info = json.loads(info_json)
    else:
        logger.info("[STATUS] No user encoded info found.")
        user_info = {'id': 'anonymous'}

    return user_info
# END METHODS
<commit_msg>-> Various fixes<commit_after>"""

Copyright 2019, Institute for Systems Biology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import logging
import base64
import json
import requests

from flask import request, jsonify

logger = logging.getLogger(__name__)

# BEGIN METHODS
def _base64_decode(encoded_str):
    # Add paddings manually if necessary.
    num_missed_paddings = 4 - len(encoded_str) % 4
    if num_missed_paddings != 4:
        encoded_str += b'=' * num_missed_paddings
    return base64.b64decode(encoded_str).decode('utf-8')

def auth_info():
    """Retrieves the authenication information from Google Cloud Endpoints."""
    encoded_info = request.headers.get('X-Endpoint-API-UserInfo', None)

    if encoded_info:
        info_json = _base64_decode(encoded_info)
        user_info = json.loads(info_json)
    else:
        logger.info("[STATUS] No user encoded info found.")
        user_info = {'id': 'anonymous', 'email': 'Anonymous'}

    return user_info
# END METHODS
<|endoftext|>


<commit_before>"""
byceps.blueprints.site.seating.views
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2020 Jochen Kupperschmidt
:License: Modified BSD, see LICENSE for details.
"""

from flask import abort, g, request

from ....services.party import service as party_service
from ....services.seating import area_service as seating_area_service
from ....services.seating.models.seat import Seat
from ....services.seating import seat_service
from ....services.seating.transfer.models import SeatID
from ....services.ticketing.models.ticket import Ticket as DbTicket
from ....services.ticketing import (
    exceptions as ticket_exceptions,
    ticket_seat_management_service,
    ticket_service,
)
from ....services.ticketing.transfer.models import TicketID
from ....util.framework.blueprint import create_blueprint
from ....util.framework.flash import flash_error, flash_success
from ....util.framework.templating import templated
from ....util.views import respond_no_content

from ...admin.seating.authorization import SeatingPermission
from ...common.authentication.decorators import login_required
from ...common.authorization.registry import permission_registry

from . import service

blueprint = create_blueprint('seating', __name__)

permission_registry.register_enum(SeatingPermission)

@blueprint.route('/')
@templated
def index():
    """List areas."""
    if g.party_id is None:
        # No party is configured for the current site.
        abort(404)

    areas = seating_area_service.get_areas_for_party(g.party_id)

    return {
        'areas': areas,
    }

@blueprint.route('/areas/<slug>')
@templated
def view_area(slug):
    """View area."""
    if g.party_id is None:
        # No party is configured for the current site.
        abort(404)

    area = seating_area_service.find_area_for_party_by_slug(g.party_id, slug)
    if area is None:
        abort(404)

    seat_management_enabled = _is_seat_management_enabled()

    seats = seat_service.get_seats_with_tickets_for_area(area.id)

    users_by_id = service.get_users(seats, [])

    seats = service.get_seats(seats, users_by_id)

    return {
        'area': area,
        'seat_management_enabled': seat_management_enabled,
        'seats': seats,
        'manage_mode': False,
    }

@blueprint.route('/areas/<slug>/manage_seats')
@login_required
@templated('seating/view_area')
def manage_seats_in_area(slug):
    """Manage seats for assigned tickets in area."""
    _abort_if_seat_management_disabled()

    area = seating_area_service.find_area_for_party_by_slug(g.party_id, slug)
    if area is None:
        abort(404)

    seat_management_enabled = _is_seat_management_enabled()

    seats = seat_service.get_seats_with_tickets_for_area(area.id)

    ticket_code = request.args.get('ticket_code')
    selected_ticket_id = request.args.get('ticket_id')

    if _is_seating_admin(g.current_user) and ticket_code:
        ticket_code = ticket_code.upper()
        ticket = ticket_service.find_ticket_by_code(ticket_code)
        if ticket:
            tickets = ticket_service.find_tickets_for_seat_manager(
                ticket.get_seat_manager().id, g.party_id
            )
            selected_ticket_id = ticket.id
        else:
            flash_error(f'Ticket code "{ticket_code}" not found.')
            tickets = []
            selected_ticket_id = None

    elif _is_seating_admin(g.current_user) and selected_ticket_id:
        ticket = ticket_service.find_ticket(selected_ticket_id)
        if ticket:
            tickets = ticket_service.find_tickets_for_seat_manager(
                ticket.get_seat_manager().id, g.party_id
            )
            selected_ticket_id = ticket.id
        else:
            flash_error(f'Ticket ID "{selected_ticket_id}" not found.')
            tickets = []
            selected_ticket_id = None

    elif seat_management_enabled:
        tickets = ticket_service.find_tickets_for_seat_manager(
            g.current_user.id, g.party_id
        )
        selected_ticket_id = None

    else:
        tickets = []
        selected_ticket_id = None

    users_by_id = service.get_users(seats, tickets)

    seats = service.get_seats(seats, users_by_id)

    if seat_management_enabled:
        managed_tickets = list(
            service.get_managed_tickets(tickets, users_by_id)
        )
    else:
        managed_tickets = []

    return {
        'area': area,
        'seat_management_enabled': seat_management_enabled,
        'seats': seats,
        'manage_mode': True,
        'managed_tickets': managed_tickets,
        'selected_ticket_id': selected_ticket_id,
    }

def _get_users(seats, managed_tickets):
    return {}  # Not implemented.

@blueprint.route(
    '/ticket/<uuid:ticket_id>/seat/<uuid:seat_id>', methods=['POST']
)
@login_required
@respond_no_content
def occupy_seat(ticket_id, seat_id):
    """Use ticket to occupy seat."""
    _abort_if_seat_management_disabled()

    ticket = _get_ticket_or_404(ticket_id)

    manager = g.current_user

    if not ticket.is_seat_managed_by(manager.id) and not _is_seating_admin(
        manager
    ):
        flash_error(
            'Du bist nicht berechtigt, den Sitzplatz '
            f'für Ticket {ticket.code} zu verwalten.'
        )
        return

    seat = _get_seat_or_404(seat_id)

    if seat.is_occupied:
        flash_error(f'{seat.label} ist bereits belegt.')
        return

    try:
        ticket_seat_management_service.occupy_seat(
            ticket.id, seat.id, manager.id
        )
    except ticket_exceptions.SeatChangeDeniedForBundledTicket:
        flash_error(
            f'Ticket {ticket.code} gehört zu einem Paket '
            'und kann nicht einzeln verwaltet werden.'
        )
        return
    except ticket_exceptions.TicketCategoryMismatch:
        flash_error(
            f'Ticket {ticket.code} und {seat.label} haben '
            'unterschiedliche Kategorien.'
        )
        return
    except ValueError:
        abort(404)

    flash_success(f'{seat.label} wurde mit Ticket {ticket.code} reserviert.')

@blueprint.route('/ticket/<uuid:ticket_id>/seat', methods=['DELETE'])
@login_required
@respond_no_content
def release_seat(ticket_id):
    """Release the seat."""
    _abort_if_seat_management_disabled()

    ticket = _get_ticket_or_404(ticket_id)

    if not ticket.occupied_seat:
        flash_error(f'Ticket {ticket.code} belegt keinen Sitzplatz.')
        return

    manager = g.current_user

    if not ticket.is_seat_managed_by(manager.id) and not _is_seating_admin(
        manager
    ):
        flash_error(
            'Du bist nicht berechtigt, den Sitzplatz '
            f'für Ticket {ticket.code} zu verwalten.'
        )
        return

    seat = ticket.occupied_seat

    try:
        ticket_seat_management_service.release_seat(ticket.id, manager.id)
    except ticket_exceptions.SeatChangeDeniedForBundledTicket:
        flash_error(
            f'Ticket {ticket.code} gehört zu einem Paket '
            'und kann nicht einzeln verwaltet werden.'
        )
        return

    flash_success(f'{seat.label} wurde freigegeben.')

def _abort_if_seat_management_disabled() -> None:
    if not _is_seat_management_enabled():
        flash_error('Sitzplätze können derzeit nicht verändert werden.')
        return

def _is_seat_management_enabled():
    if g.current_user.is_anonymous:
        return False

    if g.party_id is None:
        return False

    if _is_seating_admin(g.current_user):
        return True

    party = party_service.get_party(g.party_id)
    return party.seat_management_enabled

def _is_seating_admin(user) -> bool:
    return user.has_permission(SeatingPermission.administrate)

def _get_ticket_or_404(ticket_id: TicketID) -> DbTicket:
    ticket = ticket_service.find_ticket(ticket_id)

    if (ticket is None) or ticket.revoked:
        abort(404)

    return ticket

def _get_seat_or_404(seat_id: SeatID) -> Seat:
    seat = seat_service.find_seat(seat_id)

    if seat is None:
        abort(404)

    return seat
<commit_msg>Adjust template path to recent changes<commit_after>"""
byceps.blueprints.site.seating.views
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2020 Jochen Kupperschmidt
:License: Modified BSD, see LICENSE for details.
"""

from flask import abort, g, request

from ....services.party import service as party_service
from ....services.seating import area_service as seating_area_service
from ....services.seating.models.seat import Seat
from ....services.seating import seat_service
from ....services.seating.transfer.models import SeatID
from ....services.ticketing.models.ticket import Ticket as DbTicket
from ....services.ticketing import (
    exceptions as ticket_exceptions,
    ticket_seat_management_service,
    ticket_service,
)
from ....services.ticketing.transfer.models import TicketID
from ....util.framework.blueprint import create_blueprint
from ....util.framework.flash import flash_error, flash_success
from ....util.framework.templating import templated
from ....util.views import respond_no_content

from ...admin.seating.authorization import SeatingPermission
from ...common.authentication.decorators import login_required
from ...common.authorization.registry import permission_registry

from . import service

blueprint = create_blueprint('seating', __name__)

permission_registry.register_enum(SeatingPermission)

@blueprint.route('/')
@templated
def index():
    """List areas."""
    if g.party_id is None:
        # No party is configured for the current site.
        abort(404)

    areas = seating_area_service.get_areas_for_party(g.party_id)

    return {
        'areas': areas,
    }

@blueprint.route('/areas/<slug>')
@templated
def view_area(slug):
    """View area."""
    if g.party_id is None:
        # No party is configured for the current site.
        abort(404)

    area = seating_area_service.find_area_for_party_by_slug(g.party_id, slug)
    if area is None:
        abort(404)

    seat_management_enabled = _is_seat_management_enabled()

    seats = seat_service.get_seats_with_tickets_for_area(area.id)

    users_by_id = service.get_users(seats, [])

    seats = service.get_seats(seats, users_by_id)

    return {
        'area': area,
        'seat_management_enabled': seat_management_enabled,
        'seats': seats,
        'manage_mode': False,
    }

@blueprint.route('/areas/<slug>/manage_seats')
@login_required
@templated('site/seating/view_area')
def manage_seats_in_area(slug):
    """Manage seats for assigned tickets in area."""
    _abort_if_seat_management_disabled()

    area = seating_area_service.find_area_for_party_by_slug(g.party_id, slug)
    if area is None:
        abort(404)

    seat_management_enabled = _is_seat_management_enabled()

    seats = seat_service.get_seats_with_tickets_for_area(area.id)

    ticket_code = request.args.get('ticket_code')
    selected_ticket_id = request.args.get('ticket_id')

    if _is_seating_admin(g.current_user) and ticket_code:
        ticket_code = ticket_code.upper()
        ticket = ticket_service.find_ticket_by_code(ticket_code)
        if ticket:
            tickets = ticket_service.find_tickets_for_seat_manager(
                ticket.get_seat_manager().id, g.party_id
            )
            selected_ticket_id = ticket.id
        else:
            flash_error(f'Ticket code "{ticket_code}" not found.')
            tickets = []
            selected_ticket_id = None

    elif _is_seating_admin(g.current_user) and selected_ticket_id:
        ticket = ticket_service.find_ticket(selected_ticket_id)
        if ticket:
            tickets = ticket_service.find_tickets_for_seat_manager(
                ticket.get_seat_manager().id, g.party_id
            )
            selected_ticket_id = ticket.id
        else:
            flash_error(f'Ticket ID "{selected_ticket_id}" not found.')
            tickets = []
            selected_ticket_id = None

    elif seat_management_enabled:
        tickets = ticket_service.find_tickets_for_seat_manager(
            g.current_user.id, g.party_id
        )
        selected_ticket_id = None

    else:
        tickets = []
        selected_ticket_id = None

    users_by_id = service.get_users(seats, tickets)

    seats = service.get_seats(seats, users_by_id)

    if seat_management_enabled:
        managed_tickets = list(
            service.get_managed_tickets(tickets, users_by_id)
        )
    else:
        managed_tickets = []

    return {
        'area': area,
        'seat_management_enabled': seat_management_enabled,
        'seats': seats,
        'manage_mode': True,
        'managed_tickets': managed_tickets,
        'selected_ticket_id': selected_ticket_id,
    }

def _get_users(seats, managed_tickets):
    return {}  # Not implemented.

@blueprint.route(
    '/ticket/<uuid:ticket_id>/seat/<uuid:seat_id>', methods=['POST']
)
@login_required
@respond_no_content
def occupy_seat(ticket_id, seat_id):
    """Use ticket to occupy seat."""
    _abort_if_seat_management_disabled()

    ticket = _get_ticket_or_404(ticket_id)

    manager = g.current_user

    if not ticket.is_seat_managed_by(manager.id) and not _is_seating_admin(
        manager
    ):
        flash_error(
            'Du bist nicht berechtigt, den Sitzplatz '
            f'für Ticket {ticket.code} zu verwalten.'
        )
        return

    seat = _get_seat_or_404(seat_id)

    if seat.is_occupied:
        flash_error(f'{seat.label} ist bereits belegt.')
        return

    try:
        ticket_seat_management_service.occupy_seat(
            ticket.id, seat.id, manager.id
        )
    except ticket_exceptions.SeatChangeDeniedForBundledTicket:
        flash_error(
            f'Ticket {ticket.code} gehört zu einem Paket '
            'und kann nicht einzeln verwaltet werden.'
        )
        return
    except ticket_exceptions.TicketCategoryMismatch:
        flash_error(
            f'Ticket {ticket.code} und {seat.label} haben '
            'unterschiedliche Kategorien.'
        )
        return
    except ValueError:
        abort(404)

    flash_success(f'{seat.label} wurde mit Ticket {ticket.code} reserviert.')

@blueprint.route('/ticket/<uuid:ticket_id>/seat', methods=['DELETE'])
@login_required
@respond_no_content
def release_seat(ticket_id):
    """Release the seat."""
    _abort_if_seat_management_disabled()

    ticket = _get_ticket_or_404(ticket_id)

    if not ticket.occupied_seat:
        flash_error(f'Ticket {ticket.code} belegt keinen Sitzplatz.')
        return

    manager = g.current_user

    if not ticket.is_seat_managed_by(manager.id) and not _is_seating_admin(
        manager
    ):
        flash_error(
            'Du bist nicht berechtigt, den Sitzplatz '
            f'für Ticket {ticket.code} zu verwalten.'
        )
        return

    seat = ticket.occupied_seat

    try:
        ticket_seat_management_service.release_seat(ticket.id, manager.id)
    except ticket_exceptions.SeatChangeDeniedForBundledTicket:
        flash_error(
            f'Ticket {ticket.code} gehört zu einem Paket '
            'und kann nicht einzeln verwaltet werden.'
        )
        return

    flash_success(f'{seat.label} wurde freigegeben.')

def _abort_if_seat_management_disabled() -> None:
    if not _is_seat_management_enabled():
        flash_error('Sitzplätze können derzeit nicht verändert werden.')
        return

def _is_seat_management_enabled():
    if g.current_user.is_anonymous:
        return False

    if g.party_id is None:
        return False

    if _is_seating_admin(g.current_user):
        return True

    party = party_service.get_party(g.party_id)
    return party.seat_management_enabled

def _is_seating_admin(user) -> bool:
    return user.has_permission(SeatingPermission.administrate)

def _get_ticket_or_404(ticket_id: TicketID) -> DbTicket:
    ticket = ticket_service.find_ticket(ticket_id)

    if (ticket is None) or ticket.revoked:
        abort(404)

    return ticket

def _get_seat_or_404(seat_id: SeatID) -> Seat:
    seat = seat_service.find_seat(seat_id)

    if seat is None:
        abort(404)

    return seat
<|endoftext|>


<commit_before>"""
Large portions of this implementation were influenced or downright copied from
the `eth-testrpc` project by ConsenSys

https://github.com/ConsenSys/eth-testrpc
"""
import sys
import time
import uuid
import itertools
import functools

import gevent

import rlp

from ethereum import transactions
from ethereum import tester as t
from ethereum.utils import (
    privtoaddr,
)

from .utils import (
    is_string,
    coerce_args_to_bytes,
    strip_0x,
    encode_32bytes,
    encode_address,
    encode_data,
    normalize_number,
    normalize_address,
    decode_hex,
    mk_random_privkey,
    is_valid_block_identifier,
    is_array,
)
from .serializers import (
    serialize_txn,
    serialize_txn_receipt,
    serialize_block,
)
from .filters import (
    check_filter_topics_validity,
    process_block,
    get_filter_bounds,
)

if sys.version_info.major == 2:
    from Queue import Queue

else:
    from queue import Queue

# Set the gas
DEFAULT_GAS_LIMIT = t.gas_limit = t.GAS_LIMIT = 3141592

class EthTesterClient(object):
    """
    Stand-in replacement for the rpc client that speaks directly to the
    `ethereum.tester` facilities.
    """
    locked_accounts = None
    homestead_block_number = 0
    dao_fork_block_number = 0
    dao_fork_support = True

    def __init__(self, async=True, async_timeout=10):
        self.snapshots = []

        self.reset_evm()

        self.evm.block.config['HOMESTEAD_FORK_BLKNUM'] = self.homestead_block_number  # noqa
        self.evm.block.config['DAO_FORK_BLKNUM'] = self.dao_fork_block_number

        self.is_async = async
        self.async_timeout = async_timeout

        if self.is_async:
            self.request_queue = Queue()
            self.results = {}

            self.request_thread = gevent.spawn(self.process_requests)

        self.passphrase_accounts = {}
        self.passphrase_account_keys = {}
        self.unlocked_accounts = {}

        self.log_filters = {}
        self.log_filters_id_generator = itertools.count()

    def reset_evm(self, snapshot_idx=None):
        if snapshot_idx is not None:
            self.revert_evm(snapshot_idx)
        else:
            self.evm = t.state()
            self.evm.block.gas_limit = DEFAULT_GAS_LIMIT

    def snapshot_evm(self):
        self.snapshots.append((self.evm.block.number, self.evm.snapshot()))
        return len(self.snapshots) - 1

    def revert_evm(self, snapshot_idx=None, reset_logs=False):
        if len(self.snapshots) == 0:
            raise ValueError("No snapshots to revert to")

        if snapshot_idx is not None:
            block_number, snapshot = self.snapshots.pop(snapshot_idx)
        else:
            block_number, snapshot = self.snapshots.pop()

        # Remove all blocks after our saved block number.
        del self.evm.blocks[block_number:]

        self.evm.revert(snapshot)
        self.evm.blocks.append(self.evm.block)

    def mine_block(self):
        self.evm.mine()

    def process_requests(self):
        while True:
            id, args, kwargs = self.request_queue.get()
            mine = kwargs.pop('_mine', False)
            try:
                self._send_transaction(*args, **kwargs)
                if mine:
                    self.mine_block()
                response = self.evm.last_tx.hash
            except Exception as e:
                response = e
                if mine:
                    self.mine_block()
            self.results[id] = response

    def wait_for_block(self, block_number, max_wait=0):
        while self.evm.block.number < block_number:
            self.mine_block()
        return self.get_block_by_number(self.evm.block.number)

    def wait_for_transaction(self, txn_hash, max_wait=0):
        return self.get_transaction_receipt(txn_hash)

    def get_max_gas(self):
        return t.gas_limit

    #
    # Internal getters for EVM objects
    #
    def _get_transaction_by_hash(self, txn_hash):
        txn_hash = strip_0x(txn_hash)
        if len(txn_hash) == 64:
            txn_hash = decode_hex(txn_hash)
        for block in reversed(self.evm.blocks):
            txn_hashes = block.get_transaction_hashes()

            if txn_hash in txn_hashes:
                txn_index = txn_hashes.index(txn_hash)
                txn = block.transaction_list[txn_index]
                break
        else:
            raise ValueError("Transaction not found")
        return block, txn, txn_index

    def _get_block_by_number(self, block_number="latest"):
        if block_number == "latest":
            return self.evm.block
        elif block_number == "earliest":
            return self.evm.blocks[0]
        elif block_number == "pending":
            raise ValueError("Fetching 'pending' block is unsupported")
        else:
            block_number = normalize_number(block_number)

            if block_number >= len(self.evm.blocks):
                raise ValueError("Invalid block number")
            return self.evm.blocks[block_number]

    def _get_block_by_hash(self, block_hash):
        if len(block_hash) > 32:
            block_hash = decode_hex(strip_0x(block_hash))
        for block in self.evm.blocks:
            if block.hash == block_hash:
                return block
        else:
            raise ValueError("Could not find block for provided hash")

    @coerce_args_to_bytes
    def _send_transaction(self, _from=None, to=None, gas=None, gas_price=None,
                          value=0, data=b''):
        """
        The tester doesn't care about gas so we discard it.
        """
        if _from is None:
            _from = self.get_coinbase()

        _from = normalize_address(_from)

        try:
            sender = t.keys[t.accounts.index(_from)]
        except ValueError:
            if _from in self.unlocked_accounts:
                unlock_expiration = self.unlocked_accounts[_from]
                if unlock_expiration is None or unlock_expiration > time.time():
                    sender = self.passphrase_account_keys[_from]
                else:
                    raise ValueError("Account locked.  Unlock before sending tx")
            else:
                raise

        if to is None:
            to = b''

        to = normalize_address(to, allow_blank=True)

        if data is None:
            data = b''

        data = decode_hex(data)

        output = self.evm.send(sender=sender, to=to, value=value, evmdata=data)
        return output

    #
    # Public API
    #
    def get_coinbase(self):
        return encode_address(self.evm.block.coinbase)

    def get_accounts(self):
        return [
            encode_address(addr) for addr in t.accounts
        ] + [
            encode_address(addr) for addr in self.passphrase_accounts.keys()
        ]

    def get_code(self, address, block_number="latest"):
        block = self._get_block_by_number(block_number)
        return encode_data(block.get_code(strip_0x(address)))

    def estimate_gas(self, *args, **kwargs):
        snapshot_idx = self.snapshot_evm()

        try:
            txn_hash = self.send_transaction(*args, **kwargs)
            txn_receipt = self.get_transaction_receipt(txn_hash)
            gas_used = txn_receipt['gasUsed']
        finally:
            self.revert_evm(snapshot_idx)

        return gas_used

    def send_transaction(self, *args, **kwargs):
        if self.is_async:
            kwargs['_mine'] = True
            request_id = uuid.uuid4()
            self.request_queue.put((request_id, args, kwargs))
            start = time.time()
            while time.time() - start < self.async_timeout:
                gevent.sleep(0.1)
                if request_id in self.results:
                    result = self.results.pop(request_id)
                    if isinstance(result, Exception):
                        raise result
                    return encode_data(result)
            raise ValueError("Timeout waiting for {0}".format(request_id))
        else:
            self._send_transaction(*args, **kwargs)
            self.mine_block()
            return encode_32bytes(self.evm.last_tx.hash)

    def send_raw_transaction(self, raw_tx):
        tx = rlp.decode(decode_hex(strip_0x(raw_tx)), transactions.Transaction)

        to = encode_address(tx.to) if tx.to else b''
        _from = encode_address(tx.sender)
        data = encode_data(tx.data)

        return self.send_transaction(
            _from=_from,
            to=to,
            gas=tx.gasprice,
            value=tx.value,
            data=data,
        )

    def get_transaction_receipt(self, txn_hash):
        block, txn, txn_index = self._get_transaction_by_hash(txn_hash)

        return serialize_txn_receipt(block, txn, txn_index)

    def get_transaction_count(self, address, block_number="latest"):
        block = self._get_block_by_number(block_number)
        address = normalize_address(address)

        return block.get_nonce(address)

    def get_block_by_number(self, block_number, full_transactions=True):
        block = self._get_block_by_number(block_number)

        return serialize_block(block, full_transactions)

    def get_block_by_hash(self, block_hash, full_transactions=True):
        block = self._get_block_by_hash(block_hash)

        return serialize_block(block, full_transactions)

    def get_block_number(self):
        return self.evm.block.number

    def get_gas_price(self):
        return t.gas_price

    def get_balance(self, address, block="latest"):
        _block = self._get_block_by_number(block)
        return _block.get_balance(strip_0x(address))

    def call(self, *args, **kwargs):
        if len(args) >= 7 and args[6] != "latest":
            raise ValueError("Using call on any block other than latest is unsupported")
        if kwargs.get('block', 'latest') != "latest":
            raise ValueError("Using call on any block other than latest is unsupported")
        snapshot_idx = self.snapshot_evm()
        output = self._send_transaction(*args, **kwargs)
        self.revert_evm(snapshot_idx)

        return encode_data(output)

    def get_transaction_by_hash(self, txn_hash):
        block, txn, txn_index = self._get_transaction_by_hash(txn_hash)
        return serialize_txn(block, txn, txn_index)

    def lock_account(self, address):
        address = normalize_address(address)
        self.unlocked_accounts.pop(address, None)
        return True

    @coerce_args_to_bytes
    def check_passphrase(self, address, passphrase):
        address = normalize_address(address)
        if address not in self.passphrase_accounts:
            return False
        elif passphrase == self.passphrase_accounts[address]:
            return True
        else:
            return False

    @coerce_args_to_bytes
    def unlock_account(self, address, passphrase, duration=None):
        address = normalize_address(address)
        if self.check_passphrase(address, passphrase):
            if duration is not None:
                unlock_expiration = time.time() + duration
            else:
                unlock_expiration = None
            self.unlocked_accounts[address] = unlock_expiration
            return True
        return False

    @coerce_args_to_bytes
    def import_raw_key(self, private_key, passphrase):
        if not passphrase:
            raise ValueError("Cannot have empty passphrase")

        public_key = privtoaddr(private_key)

        self.passphrase_accounts[public_key] = passphrase
        self.passphrase_account_keys[public_key] = private_key

        return encode_address(public_key)

    @coerce_args_to_bytes
    def new_account(self, passphrase):
        private_key = mk_random_privkey()

        return self.import_raw_key(private_key, passphrase)

    @coerce_args_to_bytes
    def send_and_sign_transaction(self, passphrase, **txn_kwargs):
        try:
            _from = txn_kwargs['_from']
        except KeyError:
            raise KeyError("`send_and_sign_transaction` requires a `_from` address to be specified")  # NOQa

        _from = normalize_address(_from)

        try:
            self.unlock_account(_from, passphrase)
            return self.send_transaction(**txn_kwargs)
        finally:
            self.lock_account(_from)

    log_filters = None

    def new_filter(self, from_block="latest", to_block="latest", address=None, topics=None):
        if topics is None:
            topics = []

        if address is None:
            address = []

        if not is_valid_block_identifier(from_block):
            raise ValueError("Invalid filter parameter for `from_block`")

        if not is_valid_block_identifier(to_block):
            raise ValueError("Invalid filter parameter for `to_block`")

        if is_string(address):
            addresses = [encode_32bytes(normalize_address(address))]
        elif is_array(address):
            if not address:
                addresses = []
            elif all(is_string(addr) for addr in address):
                addresses = [encode_32bytes(normalize_address(addr)) for addr in address]
            else:
                raise ValueError(
                    "Address must be either a single address or an array of addresses"
                )
        else:
            raise ValueError(
                "Address must be either a single address or an array of addresses"
            )

        if not check_filter_topics_validity(topics):
            raise ValueError("Invalid topics")

        log_filter = {
            'from_block': from_block,
            'to_block': to_block,
            'addresses': addresses,
            'topics': topics,
        }
        filter_id = next(self.log_filters_id_generator)
        self.log_filters[filter_id] = log_filter
        return filter_id

    def new_block_filter(self, *args, **kwargs):
        raise NotImplementedError("TODO")

    def new_pending_transaction_filter(self, *args, **kwargs):
        raise NotImplementedError("TODO")

    def uninstall_filter(self, filter_id):
        try:
            self.log_filters.pop(filter_id)
        except KeyError:
            return False
        else:
            return True

    def get_filter_changes(self, filter_id):
        try:
            log_filter = self.log_filters[filter_id]
        except KeyError:
            raise ValueError("Filter not found for id: {0}".format(filter_id))

        bookmark = log_filter.get('bookmark', None)
        block_slice = get_filter_bounds(
            log_filter['from_block'],
            log_filter['to_block'],
            bookmark,
        )

        block_processor_fn = functools.partial(
            process_block,
            from_block=log_filter['from_block'],
            to_block=log_filter['to_block'],
            addresses=log_filter['addresses'],
            filter_topics=log_filter['topics'],
        )

        log_changes = list(itertools.chain.from_iterable((
            block_processor_fn(block) for block in self.evm.blocks[block_slice]
        )))

        self.log_filters[filter_id]['bookmark'] = self.evm.blocks[-1].number
        return log_changes

    def get_filter_logs(self, filter_id):
        try:
            log_filter = self.log_filters[filter_id]
        except KeyError:
            raise ValueError("Filter not found for id: {0}".format(filter_id))

        block_slice = get_filter_bounds(
            log_filter['from_block'],
            log_filter['to_block'],
        )

        block_processor_fn = functools.partial(
            process_block,
            from_block=log_filter['from_block'],
            to_block=log_filter['to_block'],
            addresses=log_filter['addresses'],
            filter_topics=log_filter['topics'],
        )

        return list(itertools.chain.from_iterable((
            block_processor_fn(block) for block in self.evm.blocks[block_slice]
        )))
<commit_msg>use gevent queues<commit_after>"""
Large portions of this implementation were influenced or downright copied from
the `eth-testrpc` project by ConsenSys

https://github.com/ConsenSys/eth-testrpc
"""
import time
import uuid
import itertools
import functools

import gevent
from gevent.queue import Queue

import rlp

from ethereum import transactions
from ethereum import tester as t
from ethereum.utils import (
    privtoaddr,
)

from .utils import (
    is_string,
    coerce_args_to_bytes,
    strip_0x,
    encode_32bytes,
    encode_address,
    encode_data,
    normalize_number,
    normalize_address,
    decode_hex,
    mk_random_privkey,
    is_valid_block_identifier,
    is_array,
)
from .serializers import (
    serialize_txn,
    serialize_txn_receipt,
    serialize_block,
)
from .filters import (
    check_filter_topics_validity,
    process_block,
    get_filter_bounds,
)

# Set the gas
DEFAULT_GAS_LIMIT = t.gas_limit = t.GAS_LIMIT = 3141592

class EthTesterClient(object):
    """
    Stand-in replacement for the rpc client that speaks directly to the
    `ethereum.tester` facilities.
    """
    locked_accounts = None
    homestead_block_number = 0
    dao_fork_block_number = 0
    dao_fork_support = True

    def __init__(self, async=True, async_timeout=10):
        self.snapshots = []

        self.reset_evm()

        self.evm.block.config['HOMESTEAD_FORK_BLKNUM'] = self.homestead_block_number  # noqa
        self.evm.block.config['DAO_FORK_BLKNUM'] = self.dao_fork_block_number

        self.is_async = async
        self.async_timeout = async_timeout

        if self.is_async:
            self.request_queue = Queue()
            self.results = {}

            self.request_thread = gevent.spawn(self.process_requests)

        self.passphrase_accounts = {}
        self.passphrase_account_keys = {}
        self.unlocked_accounts = {}

        self.log_filters = {}
        self.log_filters_id_generator = itertools.count()

    def reset_evm(self, snapshot_idx=None):
        if snapshot_idx is not None:
            self.revert_evm(snapshot_idx)
        else:
            self.evm = t.state()
            self.evm.block.gas_limit = DEFAULT_GAS_LIMIT

    def snapshot_evm(self):
        self.snapshots.append((self.evm.block.number, self.evm.snapshot()))
        return len(self.snapshots) - 1

    def revert_evm(self, snapshot_idx=None, reset_logs=False):
        if len(self.snapshots) == 0:
            raise ValueError("No snapshots to revert to")

        if snapshot_idx is not None:
            block_number, snapshot = self.snapshots.pop(snapshot_idx)
        else:
            block_number, snapshot = self.snapshots.pop()

        # Remove all blocks after our saved block number.
        del self.evm.blocks[block_number:]

        self.evm.revert(snapshot)
        self.evm.blocks.append(self.evm.block)

    def mine_block(self):
        self.evm.mine()

    def process_requests(self):
        while True:
            id, args, kwargs = self.request_queue.get()
            mine = kwargs.pop('_mine', False)
            try:
                self._send_transaction(*args, **kwargs)
                if mine:
                    self.mine_block()
                response = self.evm.last_tx.hash
            except Exception as e:
                response = e
                if mine:
                    self.mine_block()
            self.results[id] = response

    def wait_for_block(self, block_number, max_wait=0):
        while self.evm.block.number < block_number:
            self.mine_block()
        return self.get_block_by_number(self.evm.block.number)

    def wait_for_transaction(self, txn_hash, max_wait=0):
        return self.get_transaction_receipt(txn_hash)

    def get_max_gas(self):
        return t.gas_limit

    #
    # Internal getters for EVM objects
    #
    def _get_transaction_by_hash(self, txn_hash):
        txn_hash = strip_0x(txn_hash)
        if len(txn_hash) == 64:
            txn_hash = decode_hex(txn_hash)
        for block in reversed(self.evm.blocks):
            txn_hashes = block.get_transaction_hashes()

            if txn_hash in txn_hashes:
                txn_index = txn_hashes.index(txn_hash)
                txn = block.transaction_list[txn_index]
                break
        else:
            raise ValueError("Transaction not found")
        return block, txn, txn_index

    def _get_block_by_number(self, block_number="latest"):
        if block_number == "latest":
            return self.evm.block
        elif block_number == "earliest":
            return self.evm.blocks[0]
        elif block_number == "pending":
            raise ValueError("Fetching 'pending' block is unsupported")
        else:
            block_number = normalize_number(block_number)

            if block_number >= len(self.evm.blocks):
                raise ValueError("Invalid block number")
            return self.evm.blocks[block_number]

    def _get_block_by_hash(self, block_hash):
        if len(block_hash) > 32:
            block_hash = decode_hex(strip_0x(block_hash))
        for block in self.evm.blocks:
            if block.hash == block_hash:
                return block
        else:
            raise ValueError("Could not find block for provided hash")

    @coerce_args_to_bytes
    def _send_transaction(self, _from=None, to=None, gas=None, gas_price=None,
                          value=0, data=b''):
        """
        The tester doesn't care about gas so we discard it.
        """
        if _from is None:
            _from = self.get_coinbase()

        _from = normalize_address(_from)

        try:
            sender = t.keys[t.accounts.index(_from)]
        except ValueError:
            if _from in self.unlocked_accounts:
                unlock_expiration = self.unlocked_accounts[_from]
                if unlock_expiration is None or unlock_expiration > time.time():
                    sender = self.passphrase_account_keys[_from]
                else:
                    raise ValueError("Account locked.  Unlock before sending tx")
            else:
                raise

        if to is None:
            to = b''

        to = normalize_address(to, allow_blank=True)

        if data is None:
            data = b''

        data = decode_hex(data)

        output = self.evm.send(sender=sender, to=to, value=value, evmdata=data)
        return output

    #
    # Public API
    #
    def get_coinbase(self):
        return encode_address(self.evm.block.coinbase)

    def get_accounts(self):
        return [
            encode_address(addr) for addr in t.accounts
        ] + [
            encode_address(addr) for addr in self.passphrase_accounts.keys()
        ]

    def get_code(self, address, block_number="latest"):
        block = self._get_block_by_number(block_number)
        return encode_data(block.get_code(strip_0x(address)))

    def estimate_gas(self, *args, **kwargs):
        snapshot_idx = self.snapshot_evm()

        try:
            txn_hash = self.send_transaction(*args, **kwargs)
            txn_receipt = self.get_transaction_receipt(txn_hash)
            gas_used = txn_receipt['gasUsed']
        finally:
            self.revert_evm(snapshot_idx)

        return gas_used

    def send_transaction(self, *args, **kwargs):
        if self.is_async:
            kwargs['_mine'] = True
            request_id = uuid.uuid4()

            self.request_queue.put((request_id, args, kwargs))
            with gevent.Timeout(self.async_timeout):
                if request_id in self.results:
                    result = self.results.pop(request_id)
                    if isinstance(result, Exception):
                        raise result
                    return encode_data(result)
                gevent.sleep(0)
            raise ValueError("Timeout waiting for {0}".format(request_id))
        else:
            self._send_transaction(*args, **kwargs)
            self.mine_block()
            return encode_32bytes(self.evm.last_tx.hash)

    def send_raw_transaction(self, raw_tx):
        tx = rlp.decode(decode_hex(strip_0x(raw_tx)), transactions.Transaction)

        to = encode_address(tx.to) if tx.to else b''
        _from = encode_address(tx.sender)
        data = encode_data(tx.data)

        return self.send_transaction(
            _from=_from,
            to=to,
            gas=tx.gasprice,
            value=tx.value,
            data=data,
        )

    def get_transaction_receipt(self, txn_hash):
        block, txn, txn_index = self._get_transaction_by_hash(txn_hash)

        return serialize_txn_receipt(block, txn, txn_index)

    def get_transaction_count(self, address, block_number="latest"):
        block = self._get_block_by_number(block_number)
        address = normalize_address(address)

        return block.get_nonce(address)

    def get_block_by_number(self, block_number, full_transactions=True):
        block = self._get_block_by_number(block_number)

        return serialize_block(block, full_transactions)

    def get_block_by_hash(self, block_hash, full_transactions=True):
        block = self._get_block_by_hash(block_hash)

        return serialize_block(block, full_transactions)

    def get_block_number(self):
        return self.evm.block.number

    def get_gas_price(self):
        return t.gas_price

    def get_balance(self, address, block="latest"):
        _block = self._get_block_by_number(block)
        return _block.get_balance(strip_0x(address))

    def call(self, *args, **kwargs):
        if len(args) >= 7 and args[6] != "latest":
            raise ValueError("Using call on any block other than latest is unsupported")
        if kwargs.get('block', 'latest') != "latest":
            raise ValueError("Using call on any block other than latest is unsupported")
        snapshot_idx = self.snapshot_evm()
        output = self._send_transaction(*args, **kwargs)
        self.revert_evm(snapshot_idx)

        return encode_data(output)

    def get_transaction_by_hash(self, txn_hash):
        block, txn, txn_index = self._get_transaction_by_hash(txn_hash)
        return serialize_txn(block, txn, txn_index)

    def lock_account(self, address):
        address = normalize_address(address)
        self.unlocked_accounts.pop(address, None)
        return True

    @coerce_args_to_bytes
    def check_passphrase(self, address, passphrase):
        address = normalize_address(address)
        if address not in self.passphrase_accounts:
            return False
        elif passphrase == self.passphrase_accounts[address]:
            return True
        else:
            return False

    @coerce_args_to_bytes
    def unlock_account(self, address, passphrase, duration=None):
        address = normalize_address(address)
        if self.check_passphrase(address, passphrase):
            if duration is not None:
                unlock_expiration = time.time() + duration
            else:
                unlock_expiration = None
            self.unlocked_accounts[address] = unlock_expiration
            return True
        return False

    @coerce_args_to_bytes
    def import_raw_key(self, private_key, passphrase):
        if not passphrase:
            raise ValueError("Cannot have empty passphrase")

        public_key = privtoaddr(private_key)

        self.passphrase_accounts[public_key] = passphrase
        self.passphrase_account_keys[public_key] = private_key

        return encode_address(public_key)

    @coerce_args_to_bytes
    def new_account(self, passphrase):
        private_key = mk_random_privkey()

        return self.import_raw_key(private_key, passphrase)

    @coerce_args_to_bytes
    def send_and_sign_transaction(self, passphrase, **txn_kwargs):
        try:
            _from = txn_kwargs['_from']
        except KeyError:
            raise KeyError("`send_and_sign_transaction` requires a `_from` address to be specified")  # NOQa

        _from = normalize_address(_from)

        try:
            self.unlock_account(_from, passphrase)
            return self.send_transaction(**txn_kwargs)
        finally:
            self.lock_account(_from)

    log_filters = None

    def new_filter(self, from_block="latest", to_block="latest", address=None, topics=None):
        if topics is None:
            topics = []

        if address is None:
            address = []

        if not is_valid_block_identifier(from_block):
            raise ValueError("Invalid filter parameter for `from_block`")

        if not is_valid_block_identifier(to_block):
            raise ValueError("Invalid filter parameter for `to_block`")

        if is_string(address):
            addresses = [encode_32bytes(normalize_address(address))]
        elif is_array(address):
            if not address:
                addresses = []
            elif all(is_string(addr) for addr in address):
                addresses = [encode_32bytes(normalize_address(addr)) for addr in address]
            else:
                raise ValueError(
                    "Address must be either a single address or an array of addresses"
                )
        else:
            raise ValueError(
                "Address must be either a single address or an array of addresses"
            )

        if not check_filter_topics_validity(topics):
            raise ValueError("Invalid topics")

        log_filter = {
            'from_block': from_block,
            'to_block': to_block,
            'addresses': addresses,
            'topics': topics,
        }
        filter_id = next(self.log_filters_id_generator)
        self.log_filters[filter_id] = log_filter
        return filter_id

    def new_block_filter(self, *args, **kwargs):
        raise NotImplementedError("TODO")

    def new_pending_transaction_filter(self, *args, **kwargs):
        raise NotImplementedError("TODO")

    def uninstall_filter(self, filter_id):
        try:
            self.log_filters.pop(filter_id)
        except KeyError:
            return False
        else:
            return True

    def get_filter_changes(self, filter_id):
        try:
            log_filter = self.log_filters[filter_id]
        except KeyError:
            raise ValueError("Filter not found for id: {0}".format(filter_id))

        bookmark = log_filter.get('bookmark', None)
        block_slice = get_filter_bounds(
            log_filter['from_block'],
            log_filter['to_block'],
            bookmark,
        )

        block_processor_fn = functools.partial(
            process_block,
            from_block=log_filter['from_block'],
            to_block=log_filter['to_block'],
            addresses=log_filter['addresses'],
            filter_topics=log_filter['topics'],
        )

        log_changes = list(itertools.chain.from_iterable((
            block_processor_fn(block) for block in self.evm.blocks[block_slice]
        )))

        self.log_filters[filter_id]['bookmark'] = self.evm.blocks[-1].number
        return log_changes

    def get_filter_logs(self, filter_id):
        try:
            log_filter = self.log_filters[filter_id]
        except KeyError:
            raise ValueError("Filter not found for id: {0}".format(filter_id))

        block_slice = get_filter_bounds(
            log_filter['from_block'],
            log_filter['to_block'],
        )

        block_processor_fn = functools.partial(
            process_block,
            from_block=log_filter['from_block'],
            to_block=log_filter['to_block'],
            addresses=log_filter['addresses'],
            filter_topics=log_filter['topics'],
        )

        return list(itertools.chain.from_iterable((
            block_processor_fn(block) for block in self.evm.blocks[block_slice]
        )))
<|endoftext|>


<commit_before># Cabot Alert Plugin - Sensu
# --------------------------
# * Currently only works for Graphite Checks
# * Supposes you name the service as NAME@HOST so HOST can be used as source in Sensu
#
# Nicolas Truyens <nicolas@truyens.com>

from django.db import models
#from django.conf import settings
#from django.template import Context, Template

from cabot.cabotapp.alert import AlertPlugin, AlertPluginUserData

from os import environ as env

import socket
import sys
import json
#import requests

sensu_port = env.get('SENSU_PORT') or 3030              # Integer required !!!
sensu_host = env.get('SENSU_HOST') or 'localhost'
DEBUG = env.get('SENSU_DEBUG') or False

class SensuAlert(AlertPlugin):
    name = "Sensu"
    author = "Nicolas Truyens"

    def xstr(self, s):
        if s is None:
            return ''
        return str(s)

    def send_alert(self, service, users, duty_officers):
        if DEBUG:
            debug = open("/var/log/cabot/alert_sensu.log", "a")
            debug.write( 'Sending alert to Sensu.\n' )

        parts = service.name.split("@")
        if (len(parts) == 1 or len(parts) == 2):
             if (len(parts) == 1):
                source = 'network'
                checkname = service.name
             else:
                checkname = parts[0]
                source = parts[1]
        else:
            print "The service name should contain 1 @ to split up source and check name."
            if DEBUG:
                debug.write( 'The service name should contain 1 @ to split up source and check name.\n' )
            # TODO - RAISE ERROR ?
            return

        tags = list()
        tags.append(source)
        tags.append(checkname)

        if service.overall_status == service.PASSING_STATUS:
            status = '0'
        elif service.overall_status == service.WARNING_STATUS:
            status = '1'
        elif service.overall_status == service.CRITICAL_STATUS:
            status = '2'
        elif service.overall_status == service.ERROR_STATUS:
            status = '3'

        extra_info = dict()
        for check in service.all_failing_checks():
            result = check.last_result()
            for raw_data_row in json.loads(result.raw_data):
                datapoints = list()

                try:
                    #TypeError: string indices must be integers
                    datapoints_arr = raw_data_row["datapoints"]

                    if DEBUG:
                        debug.write( 'datapoints: '+datapoints_arr.to_s+'\n' )

                    for datapoint in datapoints_arr:
                        datapoints.push(datapoint[0])
                except:
                    if DEBUG:
                        debug.write( 'datapoints is not a valid dict ?\n' ) # dict() does not have a to_s function !!

            extra_info[check.name] = { 'metric': check.metric, 'took': str(result.took)+' ms', 'error': result.error, 'datapoints': datapoints }

        # Other Tags

        # REMOVED - Cabot will probably remove instances in newer releases.
#         for linked_instance in service.instances.all():
#             instance_parts = linked_instance.name.split("_")
#             for part in instance_parts:
#                 tags.append(part)

        for linked_check in service.status_checks.all():
            check_parts = linked_check.name.split("_")
            for part in check_parts:
                tags.append(part)

        hackpad = self.xstr(service.hackpad_id)
        hackpad_parts = hackpad.split(",")
        for part in hackpad_parts:
            tags.append(part)

        tags_unique = set(tags) # unique elements
        try:
            tags_unique.remove('')  # make sure there is no empty tag
        except:
            pass
        tags = list(tags_unique)    # Convert back to list (for JSON)

        output = 'Service '+service.name+': '+str(service.overall_status)
        exta_data = ', "extra_info": '+json.dumps(extra_info)+', "service_url": "'+self.xstr(service.url)+'", "tags": '+json.dumps(tags)    # Tags should be represented as array []

        if DEBUG:
            debug.write( 'source: ' + source + ' - name: ' + checkname + ' - status: ' + status + ' - output: ' + output + ' - extra_data: ' + exta_data + '\n' )

        handlerList = list()
        for user in users:
            try:
                userData = SensuAlertUserData.objects.get(user=user, title=SensuAlertUserData.name)
                if DEBUG:
                    debug.write( 'UserData: '+ userData.to_s +'\n' )  # User does not have a to_s function !!
                userHandlers = userData.handlers
                parts = userHandlers.split(",")
                for part in parts:
                    handlerList.append('"'+part+'"')
            except:
                if DEBUG:
                    debug.write( 'Error while getting userdata for user \n' ) # User does not have a to_s function !!

        uniqueHandlerList = set(handlerList)
        handlers = "[" + ",".join(uniqueHandlerList) + "]"

        if DEBUG:
            debug.write( 'handlers: ' + handlers + '\n' )
            debug.close()

        self._send_sensu_alert(source=source, check=checkname, status=status, output=output, handlers=handlers, exta_data=exta_data)

        return

    def _send_sensu_alert(self, source, check, status, output, handlers, exta_data=''):
        try:
            port = int(sensu_port)
        except ValueError:
            port = sensu_port

        ADDR = (sensu_host, port)
        DATA = '{"name": "'+check+'", "source": "'+source+'", "status": '+status+', "output": "'+output+'", "handlers": '+handlers+exta_data+' }'

        if DEBUG:
            debug = open("/var/log/cabot/alert_sensu.log", "a")

        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            if DEBUG:
                debug.write('Connecting to '+sensu_host+' on port '+str(port)+'\n')

            s.connect(ADDR)
            if DEBUG:
                debug.write('Connected!\n')

            s.send(DATA)
            if DEBUG:
                debug.write('Sent Data: '+DATA+'\n')

            s.close()
        except socket.error, msg:
            print('Exception: ' + msg + '\n')
            if DEBUG:
                debug.write('Exception: ' + msg + '\n')

        if DEBUG:
            debug.close()

class SensuAlertUserData(AlertPluginUserData):
     name = "Sensu Plugin"

     handlers = models.CharField(max_length=250, blank=True)
<commit_msg>try debugging with dir()<commit_after># Cabot Alert Plugin - Sensu
# --------------------------
# * Currently only works for Graphite Checks
# * Supposes you name the service as NAME@HOST so HOST can be used as source in Sensu
#
# Nicolas Truyens <nicolas@truyens.com>

from django.db import models
#from django.conf import settings
#from django.template import Context, Template

from cabot.cabotapp.alert import AlertPlugin, AlertPluginUserData

from os import environ as env

import socket
import sys
import json
#import requests

sensu_port = env.get('SENSU_PORT') or 3030              # Integer required !!!
sensu_host = env.get('SENSU_HOST') or 'localhost'
DEBUG = env.get('SENSU_DEBUG') or False

class SensuAlert(AlertPlugin):
    name = "Sensu"
    author = "Nicolas Truyens"

    def xstr(self, s):
        if s is None:
            return ''
        return str(s)

    def send_alert(self, service, users, duty_officers):
        if DEBUG:
            debug = open("/var/log/cabot/alert_sensu.log", "a")
            debug.write( 'Sending alert to Sensu.\n' )

        parts = service.name.split("@")
        if (len(parts) == 1 or len(parts) == 2):
             if (len(parts) == 1):
                source = 'network'
                checkname = service.name
             else:
                checkname = parts[0]
                source = parts[1]
        else:
            print "The service name should contain 1 @ to split up source and check name."
            if DEBUG:
                debug.write( 'The service name should contain 1 @ to split up source and check name.\n' )
            # TODO - RAISE ERROR ?
            return

        tags = list()
        tags.append(source)
        tags.append(checkname)

        if service.overall_status == service.PASSING_STATUS:
            status = '0'
        elif service.overall_status == service.WARNING_STATUS:
            status = '1'
        elif service.overall_status == service.CRITICAL_STATUS:
            status = '2'
        elif service.overall_status == service.ERROR_STATUS:
            status = '3'

        extra_info = dict()
        for check in service.all_failing_checks():
            result = check.last_result()
            for raw_data_row in json.loads(result.raw_data):
                datapoints = list()

                try:
                    #TypeError: string indices must be integers
                    datapoints_arr = raw_data_row["datapoints"]

                    if DEBUG:
                        debug.write( 'datapoints: '+datapoints_arr.to_s+'\n' )

                    for datapoint in datapoints_arr:
                        datapoints.push(datapoint[0])
                except:
                    if DEBUG:
                        debug.write( 'datapoints is not a valid dict ? '+dir(raw_data_row)+'\n' ) # dict() does not have a to_s function !!

            extra_info[check.name] = { 'metric': check.metric, 'took': str(result.took)+' ms', 'error': result.error, 'datapoints': datapoints }

        # Other Tags

        # REMOVED - Cabot will probably remove instances in newer releases.
#         for linked_instance in service.instances.all():
#             instance_parts = linked_instance.name.split("_")
#             for part in instance_parts:
#                 tags.append(part)

        for linked_check in service.status_checks.all():
            check_parts = linked_check.name.split("_")
            for part in check_parts:
                tags.append(part)

        hackpad = self.xstr(service.hackpad_id)
        hackpad_parts = hackpad.split(",")
        for part in hackpad_parts:
            tags.append(part)

        tags_unique = set(tags) # unique elements
        try:
            tags_unique.remove('')  # make sure there is no empty tag
        except:
            pass
        tags = list(tags_unique)    # Convert back to list (for JSON)

        output = 'Service '+service.name+': '+str(service.overall_status)
        exta_data = ', "extra_info": '+json.dumps(extra_info)+', "service_url": "'+self.xstr(service.url)+'", "tags": '+json.dumps(tags)    # Tags should be represented as array []

        if DEBUG:
            debug.write( 'source: ' + source + ' - name: ' + checkname + ' - status: ' + status + ' - output: ' + output + ' - extra_data: ' + exta_data + '\n' )

        handlerList = list()
        for user in users:
            try:
                userData = SensuAlertUserData.objects.get(user=user, title=SensuAlertUserData.name)
                if DEBUG:
                    debug.write( 'UserData: '+dir(userData)+'\n' )  # User does not have a to_s function !!
                userHandlers = userData.handlers
                parts = userHandlers.split(",")
                for part in parts:
                    handlerList.append('"'+part+'"')
            except:
                if DEBUG:
                    debug.write( 'Error while getting userdata for user '+dir(user)+'\n' ) # User does not have a to_s function !!

        uniqueHandlerList = set(handlerList)
        handlers = "[" + ",".join(uniqueHandlerList) + "]"

        if DEBUG:
            debug.write( 'handlers: ' + handlers + '\n' )
            debug.close()

        self._send_sensu_alert(source=source, check=checkname, status=status, output=output, handlers=handlers, exta_data=exta_data)

        return

    def _send_sensu_alert(self, source, check, status, output, handlers, exta_data=''):
        try:
            port = int(sensu_port)
        except ValueError:
            port = sensu_port

        ADDR = (sensu_host, port)
        DATA = '{"name": "'+check+'", "source": "'+source+'", "status": '+status+', "output": "'+output+'", "handlers": '+handlers+exta_data+' }'

        if DEBUG:
            debug = open("/var/log/cabot/alert_sensu.log", "a")

        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            if DEBUG:
                debug.write('Connecting to '+sensu_host+' on port '+str(port)+'\n')

            s.connect(ADDR)
            if DEBUG:
                debug.write('Connected!\n')

            s.send(DATA)
            if DEBUG:
                debug.write('Sent Data: '+DATA+'\n')

            s.close()
        except socket.error, msg:
            print('Exception: ' + msg + '\n')
            if DEBUG:
                debug.write('Exception: ' + msg + '\n')

        if DEBUG:
            debug.close()

class SensuAlertUserData(AlertPluginUserData):
     name = "Sensu Plugin"

     handlers = models.CharField(max_length=250, blank=True)
<|endoftext|>