<commit_before>from __future__ import absolute_import

import logging
import six

from sentry import features
from sentry.models import Activity, Event, Group, GroupStatus, Organization
from sentry.utils.http import absolute_uri
from sentry.utils.safe import safe_execute

logger = logging.getLogger('sentry.integrations.issues')

class IssueBasicMixin(object):

    def should_sync(self, attribute):
        return False

    def get_group_title(self, group, event, **kwargs):
        return event.error()

    def get_issue_url(self, key):
        """
        Given the key of the external_issue return the external issue link.
        """
        raise NotImplementedError

    def get_group_body(self, group, event, **kwargs):
        result = []
        for interface in six.itervalues(event.interfaces):
            output = safe_execute(interface.to_string, event, _with_transaction=False)
            if output:
                result.append(output)
        return '\n\n'.join(result)

    def get_group_description(self, group, event, **kwargs):
        output = [
            u'Sentry Issue: [{}]({})'.format(
                group.qualified_short_id,
                absolute_uri(group.get_absolute_url()),
            )
        ]
        body = self.get_group_body(group, event)
        if body:
            output.extend([
                '',
                '```',
                body,
                '```',
            ])
        return '\n'.join(output)

    def get_create_issue_config(self, group, **kwargs):
        """
        These fields are used to render a form for the user,
        and are then passed in the format of:

        >>>{'title': 'TypeError: Object [object Object] has no method "updateFrom"''}

        to `create_issue`, which handles creation of the issue
        in Jira, VSTS, GitHub, etc
        """
        event = group.get_latest_event()
        if event is not None:
            Event.objects.bind_nodes([event], 'data')

        return [
            {
                'name': 'title',
                'label': 'Title',
                'default': self.get_group_title(group, event, **kwargs),
                'type': 'string',
                'required': True,
            }, {
                'name': 'description',
                'label': 'Description',
                'default': self.get_group_description(group, event, **kwargs),
                'type': 'textarea',
                'autosize': True,
                'maxRows': 10,
            }
        ]

    def get_link_issue_config(self, group, **kwargs):
        """
        Used by the `GroupIntegrationDetailsEndpoint` to create an
        `ExternalIssue` using title/description obtained from calling
        `get_issue` described below.
        """
        return [
            {
                'name': 'externalIssue',
                'label': 'Issue',
                'default': '',
                'type': 'string',
            }
        ]

    def create_issue(self, data, **kwargs):
        """
        Create an issue via the provider's API and return the issue key,
        title and description.

        Should also handle API client exceptions and reraise as an
        IntegrationError (using the `message_from_error` helper).

        >>> def create_issue(self, data, **kwargs):
        >>>     resp = self.get_client().create_issue(data)
        >>>     return {
        >>>         'key': resp['id'],
        >>>         'title': resp['title'],
        >>>         'description': resp['description'],
        >>>     }
        """
        raise NotImplementedError

    def get_issue(self, issue_id, **kwargs):
        """
        Get an issue via the provider's API and return the issue key,
        title and description.

        Should also handle API client exceptions and reraise as an
        IntegrationError (using the `message_from_error` helper).

        >>> def get_issue(self, data, **kwargs):
        >>>     resp = self.get_client().get_issue(issue_id)
        >>>     return {
        >>>         'key': resp['id'],
        >>>         'title': resp['title'],
        >>>         'description': resp['description'],
        >>>     }
        """
        raise NotImplementedError

    def after_link_issue(self, external_issue, **kwargs):
        """
        Takes the external issue that has been linked via `get_issue`.

        Does anything needed after an issue has been linked, i.e. creating
        a comment for a linked issue.
        """
        pass

    def make_external_key(self, data):
        """
        Takes result of `get_issue` or `create_issue` and returns the formatted key
        """
        return data['key']

    def get_issue_display_name(self, external_issue_key):
        """
        Returns the display name of the issue.

        This is not required but helpful for integrations whose external issue key
        does not match the disired display name.
        """
        return ''

class IssueSyncMixin(IssueBasicMixin):
    comment_key = None
    outbound_status_key = None
    inbound_status_key = None
    outbound_assignee_key = None
    inbound_assignee_key = None

    def should_sync(self, attribute):
        try:
            key = getattr(self, '%s_key' % attribute)
        except AttributeError:
            return False

        if key is None:
            return False

        config = self.org_integration.config

        return config.get(key, False)

    def sync_assignee_outbound(self, external_issue, user, assign=True, **kwargs):
        """
        Propagate a sentry issue's assignee to a linked issue's assignee.
        If assign=True, we're assigning the issue. Otherwise, deassign.
        """
        raise NotImplementedError

    def sync_status_outbound(self, external_issue, is_resolved, project_id, **kwargs):
        """
        Propagate a sentry issue's status to a linked issue's status.
        """
        raise NotImplementedError

    def should_unresolve(self, data):
        """
        Given webhook data, check whether the status
        category changed FROM "done" to something else,
        meaning the sentry issue should be marked as
        unresolved

        >>> def should_unresolve(self, data):
        >>>     client = self.get_client()
        >>>     statuses = client.get_statuses()
        >>>     done_statuses = [s['id'] for s in statuses if s['category'] == 'done']
        >>>     return data['from_status'] in done_statuses \
        >>>         and data['to_status'] not in done_statuses

        """
        raise NotImplementedError

    def should_resolve(self, data):
        """
        Given webhook data, check whether the status
        category changed TO "done" from something else,
        meaning the sentry issue should be marked as
        resolved

        see example above
        """
        raise NotImplementedError

    def update_group_status(self, groups, status, activity_type):
        updated = Group.objects.filter(
            id__in=[g.id for g in groups],
        ).exclude(
            status=status,
        ).update(
            status=status,
        )
        if updated:
            for group in groups:
                activity = Activity.objects.create(
                    project=group.project,
                    group=group,
                    type=activity_type,
                )
                activity.send_notification()

    def sync_status_inbound(self, issue_key, data):
        if not self.should_sync('inbound_status'):
            return

        organization = Organization.objects.get(id=self.organization_id)
        has_issue_sync = features.has('organizations:integrations-issue-sync',
                                      organization)

        if not has_issue_sync:
            return

        affected_groups = list(
            Group.objects.get_groups_by_external_issue(
                self.model, issue_key,
            ).filter(
                project__organization_id=self.organization_id,
            ).select_related('project'),
        )

        groups_to_resolve = []
        groups_to_unresolve = []

        should_resolve = self.should_resolve(data)
        should_unresolve = self.should_unresolve(data)

        for group in affected_groups:

            # this probably shouldn't be possible unless there
            # is a bug in one of those methods
            if should_resolve is True and should_unresolve is True:
                logger.warning(
                    'sync-config-conflict', extra={
                        'organization_id': group.project.organization_id,
                        'integration_id': self.model.id,
                        'provider': self.model.get_provider(),
                    }
                )
                continue

            if should_unresolve:
                groups_to_unresolve.append(group)
            elif should_resolve:
                groups_to_resolve.append(group)

        if groups_to_resolve:
            self.update_group_status(
                groups_to_resolve,
                GroupStatus.RESOLVED,
                Activity.SET_RESOLVED,
            )

        if groups_to_unresolve:
            self.update_group_status(
                groups_to_unresolve,
                GroupStatus.UNRESOLVED,
                Activity.SET_UNRESOLVED
            )
<commit_msg>fix(integrations): Correct argument name<commit_after>from __future__ import absolute_import

import logging
import six

from sentry import features
from sentry.models import Activity, Event, Group, GroupStatus, Organization
from sentry.utils.http import absolute_uri
from sentry.utils.safe import safe_execute

logger = logging.getLogger('sentry.integrations.issues')

class IssueBasicMixin(object):

    def should_sync(self, attribute):
        return False

    def get_group_title(self, group, event, **kwargs):
        return event.error()

    def get_issue_url(self, key):
        """
        Given the key of the external_issue return the external issue link.
        """
        raise NotImplementedError

    def get_group_body(self, group, event, **kwargs):
        result = []
        for interface in six.itervalues(event.interfaces):
            output = safe_execute(interface.to_string, event, _with_transaction=False)
            if output:
                result.append(output)
        return '\n\n'.join(result)

    def get_group_description(self, group, event, **kwargs):
        output = [
            u'Sentry Issue: [{}]({})'.format(
                group.qualified_short_id,
                absolute_uri(group.get_absolute_url()),
            )
        ]
        body = self.get_group_body(group, event)
        if body:
            output.extend([
                '',
                '```',
                body,
                '```',
            ])
        return '\n'.join(output)

    def get_create_issue_config(self, group, **kwargs):
        """
        These fields are used to render a form for the user,
        and are then passed in the format of:

        >>>{'title': 'TypeError: Object [object Object] has no method "updateFrom"''}

        to `create_issue`, which handles creation of the issue
        in Jira, VSTS, GitHub, etc
        """
        event = group.get_latest_event()
        if event is not None:
            Event.objects.bind_nodes([event], 'data')

        return [
            {
                'name': 'title',
                'label': 'Title',
                'default': self.get_group_title(group, event, **kwargs),
                'type': 'string',
                'required': True,
            }, {
                'name': 'description',
                'label': 'Description',
                'default': self.get_group_description(group, event, **kwargs),
                'type': 'textarea',
                'autosize': True,
                'maxRows': 10,
            }
        ]

    def get_link_issue_config(self, group, **kwargs):
        """
        Used by the `GroupIntegrationDetailsEndpoint` to create an
        `ExternalIssue` using title/description obtained from calling
        `get_issue` described below.
        """
        return [
            {
                'name': 'externalIssue',
                'label': 'Issue',
                'default': '',
                'type': 'string',
            }
        ]

    def create_issue(self, data, **kwargs):
        """
        Create an issue via the provider's API and return the issue key,
        title and description.

        Should also handle API client exceptions and reraise as an
        IntegrationError (using the `message_from_error` helper).

        >>> def create_issue(self, data, **kwargs):
        >>>     resp = self.get_client().create_issue(data)
        >>>     return {
        >>>         'key': resp['id'],
        >>>         'title': resp['title'],
        >>>         'description': resp['description'],
        >>>     }
        """
        raise NotImplementedError

    def get_issue(self, issue_id, **kwargs):
        """
        Get an issue via the provider's API and return the issue key,
        title and description.

        Should also handle API client exceptions and reraise as an
        IntegrationError (using the `message_from_error` helper).

        >>> def get_issue(self, data, **kwargs):
        >>>     resp = self.get_client().get_issue(issue_id)
        >>>     return {
        >>>         'key': resp['id'],
        >>>         'title': resp['title'],
        >>>         'description': resp['description'],
        >>>     }
        """
        raise NotImplementedError

    def after_link_issue(self, external_issue, **kwargs):
        """
        Takes the external issue that has been linked via `get_issue`.

        Does anything needed after an issue has been linked, i.e. creating
        a comment for a linked issue.
        """
        pass

    def make_external_key(self, data):
        """
        Takes result of `get_issue` or `create_issue` and returns the formatted key
        """
        return data['key']

    def get_issue_display_name(self, external_issue):
        """
        Returns the display name of the issue.

        This is not required but helpful for integrations whose external issue key
        does not match the disired display name.
        """
        return ''

class IssueSyncMixin(IssueBasicMixin):
    comment_key = None
    outbound_status_key = None
    inbound_status_key = None
    outbound_assignee_key = None
    inbound_assignee_key = None

    def should_sync(self, attribute):
        try:
            key = getattr(self, '%s_key' % attribute)
        except AttributeError:
            return False

        if key is None:
            return False

        config = self.org_integration.config

        return config.get(key, False)

    def sync_assignee_outbound(self, external_issue, user, assign=True, **kwargs):
        """
        Propagate a sentry issue's assignee to a linked issue's assignee.
        If assign=True, we're assigning the issue. Otherwise, deassign.
        """
        raise NotImplementedError

    def sync_status_outbound(self, external_issue, is_resolved, project_id, **kwargs):
        """
        Propagate a sentry issue's status to a linked issue's status.
        """
        raise NotImplementedError

    def should_unresolve(self, data):
        """
        Given webhook data, check whether the status
        category changed FROM "done" to something else,
        meaning the sentry issue should be marked as
        unresolved

        >>> def should_unresolve(self, data):
        >>>     client = self.get_client()
        >>>     statuses = client.get_statuses()
        >>>     done_statuses = [s['id'] for s in statuses if s['category'] == 'done']
        >>>     return data['from_status'] in done_statuses \
        >>>         and data['to_status'] not in done_statuses

        """
        raise NotImplementedError

    def should_resolve(self, data):
        """
        Given webhook data, check whether the status
        category changed TO "done" from something else,
        meaning the sentry issue should be marked as
        resolved

        see example above
        """
        raise NotImplementedError

    def update_group_status(self, groups, status, activity_type):
        updated = Group.objects.filter(
            id__in=[g.id for g in groups],
        ).exclude(
            status=status,
        ).update(
            status=status,
        )
        if updated:
            for group in groups:
                activity = Activity.objects.create(
                    project=group.project,
                    group=group,
                    type=activity_type,
                )
                activity.send_notification()

    def sync_status_inbound(self, issue_key, data):
        if not self.should_sync('inbound_status'):
            return

        organization = Organization.objects.get(id=self.organization_id)
        has_issue_sync = features.has('organizations:integrations-issue-sync',
                                      organization)

        if not has_issue_sync:
            return

        affected_groups = list(
            Group.objects.get_groups_by_external_issue(
                self.model, issue_key,
            ).filter(
                project__organization_id=self.organization_id,
            ).select_related('project'),
        )

        groups_to_resolve = []
        groups_to_unresolve = []

        should_resolve = self.should_resolve(data)
        should_unresolve = self.should_unresolve(data)

        for group in affected_groups:

            # this probably shouldn't be possible unless there
            # is a bug in one of those methods
            if should_resolve is True and should_unresolve is True:
                logger.warning(
                    'sync-config-conflict', extra={
                        'organization_id': group.project.organization_id,
                        'integration_id': self.model.id,
                        'provider': self.model.get_provider(),
                    }
                )
                continue

            if should_unresolve:
                groups_to_unresolve.append(group)
            elif should_resolve:
                groups_to_resolve.append(group)

        if groups_to_resolve:
            self.update_group_status(
                groups_to_resolve,
                GroupStatus.RESOLVED,
                Activity.SET_RESOLVED,
            )

        if groups_to_unresolve:
            self.update_group_status(
                groups_to_unresolve,
                GroupStatus.UNRESOLVED,
                Activity.SET_UNRESOLVED
            )
<|endoftext|>


<commit_before>"""
sentry.web.frontend.groups
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

import datetime
import logging
import re

from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.safestring import mark_safe

from sentry.conf import settings
from sentry.constants import (SORT_OPTIONS, SEARCH_SORT_OPTIONS,
    SORT_CLAUSES, MYSQL_SORT_CLAUSES, SQLITE_SORT_CLAUSES, MEMBER_USER,
    SCORE_CLAUSES, MYSQL_SCORE_CLAUSES, SQLITE_SCORE_CLAUSES)
from sentry.filters import get_filters
from sentry.models import Group, Event, View, SearchDocument
from sentry.permissions import can_admin_group
from sentry.plugins import plugins
from sentry.utils import json
from sentry.utils.dates import parse_date
from sentry.utils.db import has_trending, get_db_engine
from sentry.web.decorators import has_access, has_group_access, login_required
from sentry.web.helpers import render_to_response

uuid_re = re.compile(r'^[a-z0-9]{32}$', re.I)
event_re = re.compile(r'^(?P<event_id>[a-z0-9]{32})\$(?P<checksum>[a-z0-9]{32})$', re.I)

def _get_rendered_interfaces(event):
    interface_list = []
    for interface in event.interfaces.itervalues():
        try:
            html = interface.to_html(event)
        except Exception:
            logger = logging.getLogger('sentry.interfaces')
            logger.error('Error rendering interface %r', interface.__class__, extra={
                'event_id': event.id,
            }, exc_info=True)
            continue
        if not html:
            continue
        interface_list.append(mark_safe(html))
    return interface_list

def _get_group_list(request, project, view=None):
    filters = []
    for cls in get_filters(Group, project):
        try:
            filters.append(cls(request, project))
        except Exception, e:
            logger = logging.getLogger('sentry.filters')
            logger.exception('Error initializing filter %r: %s', cls, e)

    event_list = Group.objects
    if request.user.is_authenticated() and request.GET.get('bookmarks'):
        event_list = event_list.filter(
            bookmark_set__project=project,
            bookmark_set__user=request.user,
        )
    else:
        event_list = event_list.filter(project=project)

    if view:
        event_list = event_list.filter(views=view)

    for filter_ in filters:
        try:
            if not filter_.is_set():
                continue
            event_list = filter_.get_query_set(event_list)
        except Exception, e:
            logger = logging.getLogger('sentry.filters')
            logger.exception('Error processing filter %r: %s', cls, e)

    date_from = request.GET.get('df')
    time_from = request.GET.get('tf')
    date_to = request.GET.get('dt')
    time_to = request.GET.get('tt')

    today = timezone.now()

    # date format is Y-m-d
    if any(x is not None for x in [date_from, time_from, date_to, time_to]):
        date_from, date_to = parse_date(date_from, time_from), parse_date(date_to, time_to)
    else:
        date_from = today - datetime.timedelta(days=3)
        date_to = None

    if date_from and date_to:
        event_list = event_list.filter(
            messagecountbyminute__date__gte=date_from,
            messagecountbyminute__date__lte=date_to,
        )
    elif date_from:
        event_list = event_list.filter(last_seen__gte=date_from)
    elif date_to:
        event_list = event_list.filter(last_seen__lte=date_to)

    sort = request.GET.get('sort')
    if sort not in SORT_OPTIONS:
        sort = settings.DEFAULT_SORT_OPTION

    if sort.startswith('accel_') and not has_trending():
        sort = settings.DEFAULT_SORT_OPTION

    engine = get_db_engine('default')
    if engine.startswith('sqlite'):
        score_clause = SQLITE_SORT_CLAUSES.get(sort)
        filter_clause = SQLITE_SCORE_CLAUSES.get(sort)
    elif engine.startswith('mysql'):
        score_clause = MYSQL_SORT_CLAUSES.get(sort)
        filter_clause = MYSQL_SCORE_CLAUSES.get(sort)
    else:
        score_clause = SORT_CLAUSES.get(sort)
        filter_clause = SCORE_CLAUSES.get(sort)

    # All filters must already be applied once we reach this point
    if sort == 'tottime':
        event_list = event_list.filter(time_spent_count__gt=0)
    elif sort == 'avgtime':
        event_list = event_list.filter(time_spent_count__gt=0)
    elif sort.startswith('accel_'):
        event_list = Group.objects.get_accelerated([project.id], event_list, minutes=int(sort.split('_', 1)[1]))

    if score_clause:
        event_list = event_list.extra(
            select={'sort_value': score_clause},
        ).order_by('-sort_value', '-last_seen')
        cursor = request.GET.get('cursor')
        if cursor:
            event_list = event_list.extra(
                where=['%s > %%s' % filter_clause],
                params=[cursor],
            )

    return {
        'filters': filters,
        'event_list': event_list,
        'date_from': date_from,
        'date_to': date_to,
        'today': today,
        'sort': sort,
    }

@login_required
@has_access
def search(request, project):
    query = request.GET.get('q')

    if not query:
        return HttpResponseRedirect(reverse('sentry', args=[project.slug]))

    sort = request.GET.get('sort')
    if sort not in SEARCH_SORT_OPTIONS:
        sort = settings.SEARCH_DEFAULT_SORT_OPTION
    sort_label = SEARCH_SORT_OPTIONS[sort]

    result = event_re.match(query)
    if result:
        # Forward to message if it exists
        # event_id = result.group(1)
        checksum = result.group(2)
        event_list = Group.objects.filter(project=project, checksum=checksum)
        top_matches = list(event_list[:2])
        if len(top_matches) == 0:
            return render_to_response('sentry/invalid_message_id.html', {
                'project': project,
            }, request)
        elif len(top_matches) == 1:
            return HttpResponseRedirect(top_matches[0].get_absolute_url())
    elif uuid_re.match(query):
        # Forward to message if it exists
        try:
            message = Event.objects.get(project=project, event_id=query)
        except Event.DoesNotExist:
            return render_to_response('sentry/invalid_message_id.html', {
                'project': project,
            }, request)
        else:
            return HttpResponseRedirect(message.get_absolute_url())
    elif not settings.USE_SEARCH:
        event_list = Group.objects.none()
        # return render_to_response('sentry/invalid_message_id.html', {
        #         'project': project,
        #     }, request)
    else:
        documents = list(SearchDocument.objects.search(project, query, sort_by=sort))
        groups = Group.objects.in_bulk([d.group_id for d in documents])

        event_list = []
        for doc in documents:
            try:
                event_list.append(groups[doc.group_id])
            except KeyError:
                continue

    return render_to_response('sentry/search.html', {
        'project': project,
        'event_list': event_list,
        'query': query,
        'sort': sort,
        'sort_label': sort_label,
    }, request)

@login_required
@has_access
def group_list(request, project):
    try:
        page = int(request.GET.get('p', 1))
    except (TypeError, ValueError):
        page = 1

    view_id = request.GET.get('view')
    if view_id:
        try:
            view = View.objects.get_from_cache(pk=int(view_id))
        except View.DoesNotExist:
            return HttpResponseRedirect(reverse('sentry', args=[project.slug]))
    else:
        view = None

    response = _get_group_list(
        request=request,
        project=project,
        view=view,
    )

    # XXX: this is duplicate in _get_group_list
    sort_label = SORT_OPTIONS[response['sort']]

    has_realtime = page == 1

    return render_to_response('sentry/groups/group_list.html', {
        'project': project,
        'from_date': response['date_from'],
        'to_date': response['date_to'],
        'has_realtime': has_realtime,
        'event_list': response['event_list'],
        'today': response['today'],
        'sort': response['sort'],
        'sort_label': sort_label,
        'filters': response['filters'],
        'view': view,
        'SORT_OPTIONS': SORT_OPTIONS,
        'HAS_TRENDING': has_trending(),
        'PAGE': 'dashboard',
    }, request)

@has_group_access
def group(request, project, group):
    # It's possible that a message would not be created under certain
    # circumstances (such as a post_save signal failing)
    event = group.get_latest_event() or Event(group=group)

    return render_to_response('sentry/groups/details.html', {
        'project': project,
        'page': 'details',
        'group': group,
        'event': event,
        'interface_list': _get_rendered_interfaces(event),
        'json_data': event.data.get('extra', {}),
        'version_data': event.data.get('modules', None),
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_group_access
def group_tag_details(request, project, group, tag_name):
    return render_to_response('sentry/plugins/bases/tag/index.html', {
        'project': project,
        'group': group,
        'title': tag_name.replace('_', ' ').title(),
        'tag_name': tag_name,
        'unique_tags': group.get_unique_tags(tag_name),
        'group': group,
        'page': 'tag_details',
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_group_access
def group_event_list(request, project, group):
    event_list = group.event_set.all().order_by('-datetime')

    return render_to_response('sentry/groups/event_list.html', {
        'project': project,
        'group': group,
        'event_list': event_list,
        'page': 'event_list',
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_access(MEMBER_USER)
def group_event_list_json(request, project, group_id):
    group = get_object_or_404(Group, pk=group_id, project=project)

    limit = request.GET.get('limit', settings.MAX_JSON_RESULTS)
    try:
        limit = int(limit)
    except ValueError:
        return HttpResponse('non numeric limit', status=400, mimetype='text/plain')
    if limit > settings.MAX_JSON_RESULTS:
        return HttpResponse("too many objects requested", mimetype='text/plain', status=400)

    events = group.event_set.order_by('-id')[:limit]

    return HttpResponse(json.dumps([event.as_dict() for event in events]), mimetype='application/json')

@has_group_access
def group_event_details(request, project, group, event_id):
    event = get_object_or_404(group.event_set, pk=event_id)

    return render_to_response('sentry/groups/event.html', {
        'project': project,
        'page': 'event_list',
        'group': group,
        'event': event,
        'interface_list': _get_rendered_interfaces(event),
        'json_data': event.data.get('extra', {}),
        'version_data': event.data.get('modules', None),
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_access(MEMBER_USER)
def group_event_details_json(request, project, group_id, event_id_or_latest):
    group = get_object_or_404(Group, pk=group_id, project=project)

    if event_id_or_latest == 'latest':
        # It's possible that a message would not be created under certain
        # circumstances (such as a post_save signal failing)
        event = group.get_latest_event() or Event()
    else:
        event = get_object_or_404(group.event_set, pk=event_id_or_latest)

    return HttpResponse(json.dumps(event.as_dict()), mimetype='application/json')

@login_required
@has_access(MEMBER_USER)
def group_plugin_action(request, project, group_id, slug):
    group = get_object_or_404(Group, pk=group_id, project=project)

    try:
        plugin = plugins.get(slug)
    except KeyError:
        raise Http404('Plugin not found')

    response = plugin.get_view_response(request, group)
    if response:
        return response
    return HttpResponseRedirect(request.META.get('HTTP_REFERER') or reverse('sentry', kwargs={'project_id': group.project.slug}))
<commit_msg>Eliminate excess project query<commit_after>"""
sentry.web.frontend.groups
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

import datetime
import logging
import re

from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.safestring import mark_safe

from sentry.conf import settings
from sentry.constants import (SORT_OPTIONS, SEARCH_SORT_OPTIONS,
    SORT_CLAUSES, MYSQL_SORT_CLAUSES, SQLITE_SORT_CLAUSES, MEMBER_USER,
    SCORE_CLAUSES, MYSQL_SCORE_CLAUSES, SQLITE_SCORE_CLAUSES)
from sentry.filters import get_filters
from sentry.models import Group, Event, View, SearchDocument
from sentry.permissions import can_admin_group
from sentry.plugins import plugins
from sentry.utils import json
from sentry.utils.dates import parse_date
from sentry.utils.db import has_trending, get_db_engine
from sentry.web.decorators import has_access, has_group_access, login_required
from sentry.web.helpers import render_to_response

uuid_re = re.compile(r'^[a-z0-9]{32}$', re.I)
event_re = re.compile(r'^(?P<event_id>[a-z0-9]{32})\$(?P<checksum>[a-z0-9]{32})$', re.I)

def _get_rendered_interfaces(event):
    interface_list = []
    for interface in event.interfaces.itervalues():
        try:
            html = interface.to_html(event)
        except Exception:
            logger = logging.getLogger('sentry.interfaces')
            logger.error('Error rendering interface %r', interface.__class__, extra={
                'event_id': event.id,
            }, exc_info=True)
            continue
        if not html:
            continue
        interface_list.append(mark_safe(html))
    return interface_list

def _get_group_list(request, project, view=None):
    filters = []
    for cls in get_filters(Group, project):
        try:
            filters.append(cls(request, project))
        except Exception, e:
            logger = logging.getLogger('sentry.filters')
            logger.exception('Error initializing filter %r: %s', cls, e)

    event_list = Group.objects
    if request.user.is_authenticated() and request.GET.get('bookmarks'):
        event_list = event_list.filter(
            bookmark_set__project=project,
            bookmark_set__user=request.user,
        )
    else:
        event_list = event_list.filter(project=project)

    if view:
        event_list = event_list.filter(views=view)

    for filter_ in filters:
        try:
            if not filter_.is_set():
                continue
            event_list = filter_.get_query_set(event_list)
        except Exception, e:
            logger = logging.getLogger('sentry.filters')
            logger.exception('Error processing filter %r: %s', cls, e)

    date_from = request.GET.get('df')
    time_from = request.GET.get('tf')
    date_to = request.GET.get('dt')
    time_to = request.GET.get('tt')

    today = timezone.now()

    # date format is Y-m-d
    if any(x is not None for x in [date_from, time_from, date_to, time_to]):
        date_from, date_to = parse_date(date_from, time_from), parse_date(date_to, time_to)
    else:
        date_from = today - datetime.timedelta(days=3)
        date_to = None

    if date_from and date_to:
        event_list = event_list.filter(
            messagecountbyminute__date__gte=date_from,
            messagecountbyminute__date__lte=date_to,
        )
    elif date_from:
        event_list = event_list.filter(last_seen__gte=date_from)
    elif date_to:
        event_list = event_list.filter(last_seen__lte=date_to)

    sort = request.GET.get('sort')
    if sort not in SORT_OPTIONS:
        sort = settings.DEFAULT_SORT_OPTION

    if sort.startswith('accel_') and not has_trending():
        sort = settings.DEFAULT_SORT_OPTION

    engine = get_db_engine('default')
    if engine.startswith('sqlite'):
        score_clause = SQLITE_SORT_CLAUSES.get(sort)
        filter_clause = SQLITE_SCORE_CLAUSES.get(sort)
    elif engine.startswith('mysql'):
        score_clause = MYSQL_SORT_CLAUSES.get(sort)
        filter_clause = MYSQL_SCORE_CLAUSES.get(sort)
    else:
        score_clause = SORT_CLAUSES.get(sort)
        filter_clause = SCORE_CLAUSES.get(sort)

    # All filters must already be applied once we reach this point
    if sort == 'tottime':
        event_list = event_list.filter(time_spent_count__gt=0)
    elif sort == 'avgtime':
        event_list = event_list.filter(time_spent_count__gt=0)
    elif sort.startswith('accel_'):
        event_list = Group.objects.get_accelerated([project.id], event_list, minutes=int(sort.split('_', 1)[1]))

    if score_clause:
        event_list = event_list.extra(
            select={'sort_value': score_clause},
        ).order_by('-sort_value', '-last_seen')
        cursor = request.GET.get('cursor')
        if cursor:
            event_list = event_list.extra(
                where=['%s > %%s' % filter_clause],
                params=[cursor],
            )

    event_list = event_list.select_related('project')

    return {
        'filters': filters,
        'event_list': event_list,
        'date_from': date_from,
        'date_to': date_to,
        'today': today,
        'sort': sort,
    }

@login_required
@has_access
def search(request, project):
    query = request.GET.get('q')

    if not query:
        return HttpResponseRedirect(reverse('sentry', args=[project.slug]))

    sort = request.GET.get('sort')
    if sort not in SEARCH_SORT_OPTIONS:
        sort = settings.SEARCH_DEFAULT_SORT_OPTION
    sort_label = SEARCH_SORT_OPTIONS[sort]

    result = event_re.match(query)
    if result:
        # Forward to message if it exists
        # event_id = result.group(1)
        checksum = result.group(2)
        event_list = Group.objects.filter(project=project, checksum=checksum)
        top_matches = list(event_list[:2])
        if len(top_matches) == 0:
            return render_to_response('sentry/invalid_message_id.html', {
                'project': project,
            }, request)
        elif len(top_matches) == 1:
            return HttpResponseRedirect(top_matches[0].get_absolute_url())
    elif uuid_re.match(query):
        # Forward to message if it exists
        try:
            message = Event.objects.get(project=project, event_id=query)
        except Event.DoesNotExist:
            return render_to_response('sentry/invalid_message_id.html', {
                'project': project,
            }, request)
        else:
            return HttpResponseRedirect(message.get_absolute_url())
    elif not settings.USE_SEARCH:
        event_list = Group.objects.none()
        # return render_to_response('sentry/invalid_message_id.html', {
        #         'project': project,
        #     }, request)
    else:
        documents = list(SearchDocument.objects.search(project, query, sort_by=sort))
        groups = Group.objects.in_bulk([d.group_id for d in documents])

        event_list = []
        for doc in documents:
            try:
                event_list.append(groups[doc.group_id])
            except KeyError:
                continue

    return render_to_response('sentry/search.html', {
        'project': project,
        'event_list': event_list,
        'query': query,
        'sort': sort,
        'sort_label': sort_label,
    }, request)

@login_required
@has_access
def group_list(request, project):
    try:
        page = int(request.GET.get('p', 1))
    except (TypeError, ValueError):
        page = 1

    view_id = request.GET.get('view')
    if view_id:
        try:
            view = View.objects.get_from_cache(pk=int(view_id))
        except View.DoesNotExist:
            return HttpResponseRedirect(reverse('sentry', args=[project.slug]))
    else:
        view = None

    response = _get_group_list(
        request=request,
        project=project,
        view=view,
    )

    # XXX: this is duplicate in _get_group_list
    sort_label = SORT_OPTIONS[response['sort']]

    has_realtime = page == 1

    return render_to_response('sentry/groups/group_list.html', {
        'project': project,
        'from_date': response['date_from'],
        'to_date': response['date_to'],
        'has_realtime': has_realtime,
        'event_list': response['event_list'],
        'today': response['today'],
        'sort': response['sort'],
        'sort_label': sort_label,
        'filters': response['filters'],
        'view': view,
        'SORT_OPTIONS': SORT_OPTIONS,
        'HAS_TRENDING': has_trending(),
        'PAGE': 'dashboard',
    }, request)

@has_group_access
def group(request, project, group):
    # It's possible that a message would not be created under certain
    # circumstances (such as a post_save signal failing)
    event = group.get_latest_event() or Event(group=group)

    return render_to_response('sentry/groups/details.html', {
        'project': project,
        'page': 'details',
        'group': group,
        'event': event,
        'interface_list': _get_rendered_interfaces(event),
        'json_data': event.data.get('extra', {}),
        'version_data': event.data.get('modules', None),
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_group_access
def group_tag_details(request, project, group, tag_name):
    return render_to_response('sentry/plugins/bases/tag/index.html', {
        'project': project,
        'group': group,
        'title': tag_name.replace('_', ' ').title(),
        'tag_name': tag_name,
        'unique_tags': group.get_unique_tags(tag_name),
        'group': group,
        'page': 'tag_details',
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_group_access
def group_event_list(request, project, group):
    event_list = group.event_set.all().order_by('-datetime')

    return render_to_response('sentry/groups/event_list.html', {
        'project': project,
        'group': group,
        'event_list': event_list,
        'page': 'event_list',
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_access(MEMBER_USER)
def group_event_list_json(request, project, group_id):
    group = get_object_or_404(Group, pk=group_id, project=project)

    limit = request.GET.get('limit', settings.MAX_JSON_RESULTS)
    try:
        limit = int(limit)
    except ValueError:
        return HttpResponse('non numeric limit', status=400, mimetype='text/plain')
    if limit > settings.MAX_JSON_RESULTS:
        return HttpResponse("too many objects requested", mimetype='text/plain', status=400)

    events = group.event_set.order_by('-id')[:limit]

    return HttpResponse(json.dumps([event.as_dict() for event in events]), mimetype='application/json')

@has_group_access
def group_event_details(request, project, group, event_id):
    event = get_object_or_404(group.event_set, pk=event_id)

    return render_to_response('sentry/groups/event.html', {
        'project': project,
        'page': 'event_list',
        'group': group,
        'event': event,
        'interface_list': _get_rendered_interfaces(event),
        'json_data': event.data.get('extra', {}),
        'version_data': event.data.get('modules', None),
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_access(MEMBER_USER)
def group_event_details_json(request, project, group_id, event_id_or_latest):
    group = get_object_or_404(Group, pk=group_id, project=project)

    if event_id_or_latest == 'latest':
        # It's possible that a message would not be created under certain
        # circumstances (such as a post_save signal failing)
        event = group.get_latest_event() or Event()
    else:
        event = get_object_or_404(group.event_set, pk=event_id_or_latest)

    return HttpResponse(json.dumps(event.as_dict()), mimetype='application/json')

@login_required
@has_access(MEMBER_USER)
def group_plugin_action(request, project, group_id, slug):
    group = get_object_or_404(Group, pk=group_id, project=project)

    try:
        plugin = plugins.get(slug)
    except KeyError:
        raise Http404('Plugin not found')

    response = plugin.get_view_response(request, group)
    if response:
        return response
    return HttpResponseRedirect(request.META.get('HTTP_REFERER') or reverse('sentry', kwargs={'project_id': group.project.slug}))
<|endoftext|>


<commit_before>"""
sentry.web.frontend.groups
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

import datetime
import logging
import re

from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.safestring import mark_safe

from sentry.conf import settings
from sentry.constants import (SORT_OPTIONS, SEARCH_SORT_OPTIONS,
    SORT_CLAUSES, MYSQL_SORT_CLAUSES, SQLITE_SORT_CLAUSES, MEMBER_USER,
    SCORE_CLAUSES, MYSQL_SCORE_CLAUSES, SQLITE_SCORE_CLAUSES)
from sentry.filters import get_filters
from sentry.models import Group, Event, View, SearchDocument
from sentry.permissions import can_admin_group
from sentry.plugins import plugins
from sentry.utils import json
from sentry.utils.dates import parse_date
from sentry.utils.db import has_trending, get_db_engine
from sentry.web.decorators import has_access, has_group_access, login_required
from sentry.web.helpers import render_to_response

uuid_re = re.compile(r'^[a-z0-9]{32}$', re.I)
event_re = re.compile(r'^(?P<event_id>[a-z0-9]{32})\$(?P<checksum>[a-z0-9]{32})$', re.I)

def _get_rendered_interfaces(event):
    interface_list = []
    for interface in event.interfaces.itervalues():
        try:
            html = interface.to_html(event)
        except Exception:
            logger = logging.getLogger('sentry.interfaces')
            logger.exception('Error rendering interface %r', interface.__class__, extra={
                'event_id': event.id,
            })
            continue
        if not html:
            continue
        interface_list.append(mark_safe(html))
    return interface_list

def _get_group_list(request, project, view=None):
    filters = []
    for cls in get_filters(Group, project):
        try:
            filters.append(cls(request, project))
        except Exception, e:
            logger = logging.getLogger('sentry.filters')
            logger.exception('Error initializing filter %r: %s', cls, e)

    event_list = Group.objects
    if request.user.is_authenticated() and request.GET.get('bookmarks'):
        event_list = event_list.filter(
            bookmark_set__project=project,
            bookmark_set__user=request.user,
        )
    else:
        event_list = event_list.filter(project=project)

    if view:
        event_list = event_list.filter(views=view)

    for filter_ in filters:
        try:
            if not filter_.is_set():
                continue
            event_list = filter_.get_query_set(event_list)
        except Exception, e:
            logger = logging.getLogger('sentry.filters')
            logger.exception('Error processing filter %r: %s', cls, e)

    date_from = request.GET.get('df')
    time_from = request.GET.get('tf')
    date_to = request.GET.get('dt')
    time_to = request.GET.get('tt')

    today = timezone.now()

    # date format is Y-m-d
    if any(x is not None for x in [date_from, time_from, date_to, time_to]):
        date_from, date_to = parse_date(date_from, time_from), parse_date(date_to, time_to)
    else:
        date_from = today - datetime.timedelta(days=3)
        date_to = None

    if date_from:
        if not date_to:
            event_list = event_list.filter(last_seen__gte=date_from)
        else:
            event_list = event_list.filter(messagecountbyminute__date__gte=date_from)
    if date_to:
        if not date_from:
            event_list = event_list.filter(last_seen__lte=date_to)
        else:
            event_list = event_list.filter(messagecountbyminute__date__lte=date_to)

    sort = request.GET.get('sort')
    if sort not in SORT_OPTIONS:
        sort = settings.DEFAULT_SORT_OPTION

    if sort.startswith('accel_') and not has_trending():
        sort = settings.DEFAULT_SORT_OPTION

    engine = get_db_engine('default')
    if engine.startswith('sqlite'):
        score_clause = SQLITE_SORT_CLAUSES.get(sort)
        filter_clause = SQLITE_SCORE_CLAUSES.get(sort)
    elif engine.startswith('mysql'):
        score_clause = MYSQL_SORT_CLAUSES.get(sort)
        filter_clause = MYSQL_SCORE_CLAUSES.get(sort)
    else:
        score_clause = SORT_CLAUSES.get(sort)
        filter_clause = SCORE_CLAUSES.get(sort)

    # All filters must already be applied once we reach this point
    if sort == 'tottime':
        event_list = event_list.filter(time_spent_count__gt=0)
    elif sort == 'avgtime':
        event_list = event_list.filter(time_spent_count__gt=0)
    elif sort.startswith('accel_'):
        event_list = Group.objects.get_accelerated(event_list, minutes=int(sort.split('_', 1)[1]))

    if score_clause:
        event_list = event_list.extra(
            select={'sort_value': score_clause},
        ).order_by('-sort_value', '-last_seen')
        cursor = request.GET.get('cursor')
        if cursor:
            event_list = event_list.extra(
                where=['%s > %%s' % filter_clause],
                params=[cursor],
            )

    return {
        'filters': filters,
        'event_list': event_list,
        'date_from': date_from,
        'date_to': date_to,
        'today': today,
        'sort': sort,
    }

@login_required
@has_access
def search(request, project):
    query = request.GET.get('q')

    if not query:
        return HttpResponseRedirect(reverse('sentry', args=[project.slug]))

    sort = request.GET.get('sort')
    if sort not in SEARCH_SORT_OPTIONS:
        sort = settings.SEARCH_DEFAULT_SORT_OPTION
    sort_label = SEARCH_SORT_OPTIONS[sort]

    result = event_re.match(query)
    if result:
        # Forward to message if it exists
        # event_id = result.group(1)
        checksum = result.group(2)
        event_list = Group.objects.filter(checksum=checksum)
        top_matches = event_list[:2]
        if len(top_matches) == 0:
            return render_to_response('sentry/invalid_message_id.html', {
                'project': project,
            }, request)
        elif len(top_matches) == 1:
            return HttpResponseRedirect(top_matches[0].get_absolute_url())
    elif uuid_re.match(query):
        # Forward to message if it exists
        try:
            message = Event.objects.get(project=project, event_id=query)
        except Event.DoesNotExist:
            return render_to_response('sentry/invalid_message_id.html', {
                'project': project,
            }, request)
        else:
            return HttpResponseRedirect(message.get_absolute_url())
    elif not settings.USE_SEARCH:
        event_list = Group.objects.none()
        # return render_to_response('sentry/invalid_message_id.html', {
        #         'project': project,
        #     }, request)
    else:
        documents = list(SearchDocument.objects.search(project, query, sort_by=sort))
        groups = Group.objects.in_bulk([d.group_id for d in documents])

        event_list = []
        for doc in documents:
            try:
                event_list.append(groups[doc.group_id])
            except KeyError:
                continue

    return render_to_response('sentry/search.html', {
        'project': project,
        'event_list': event_list,
        'query': query,
        'sort': sort,
        'sort_label': sort_label,
    }, request)

@login_required
@has_access
def group_list(request, project):
    try:
        page = int(request.GET.get('p', 1))
    except (TypeError, ValueError):
        page = 1

    view_id = request.GET.get('view')
    if view_id:
        try:
            view = View.objects.get_from_cache(pk=int(view_id))
        except View.DoesNotExist:
            return HttpResponseRedirect(reverse('sentry', args=[project.slug]))
    else:
        view = None

    response = _get_group_list(
        request=request,
        project=project,
        view=view,
    )

    # XXX: this is duplicate in _get_group_list
    sort_label = SORT_OPTIONS[response['sort']]

    has_realtime = page == 1

    return render_to_response('sentry/groups/group_list.html', {
        'project': project,
        'from_date': response['date_from'],
        'to_date': response['date_to'],
        'has_realtime': has_realtime,
        'event_list': response['event_list'],
        'today': response['today'],
        'sort': response['sort'],
        'sort_label': sort_label,
        'filters': response['filters'],
        'view': view,
        'SORT_OPTIONS': SORT_OPTIONS,
        'HAS_TRENDING': has_trending(),
        'PAGE': 'dashboard',
    }, request)

@has_group_access
def group(request, project, group):
    # It's possible that a message would not be created under certain
    # circumstances (such as a post_save signal failing)
    event = group.get_latest_event() or Event(group=group)

    return render_to_response('sentry/groups/details.html', {
        'project': project,
        'page': 'details',
        'group': group,
        'event': event,
        'interface_list': _get_rendered_interfaces(event),
        'json_data': event.data.get('extra', {}),
        'version_data': event.data.get('modules', None),
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_group_access
def group_tag_details(request, project, group, tag_name):
    return render_to_response('sentry/plugins/bases/tag/index.html', {
        'project': project,
        'group': group,
        'title': tag_name.replace('_', ' ').title(),
        'tag_name': tag_name,
        'unique_tags': group.get_unique_tags(tag_name),
        'group': group,
        'page': 'tag_details',
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_group_access
def group_event_list(request, project, group):
    event_list = group.event_set.all().order_by('-datetime')

    return render_to_response('sentry/groups/event_list.html', {
        'project': project,
        'group': group,
        'event_list': event_list,
        'page': 'event_list',
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_access(MEMBER_USER)
def group_event_list_json(request, project, group_id):
    group = get_object_or_404(Group, pk=group_id, project=project)

    limit = request.GET.get('limit', settings.MAX_JSON_RESULTS)
    try:
        limit = int(limit)
    except ValueError:
        return HttpResponse('non numeric limit', status=400, mimetype='text/plain')
    if limit > settings.MAX_JSON_RESULTS:
        return HttpResponse("too many objects requested", mimetype='text/plain', status=400)

    events = group.event_set.order_by('-id')[:limit]

    return HttpResponse(json.dumps([event.as_dict() for event in events]), mimetype='application/json')

@has_group_access
def group_event_details(request, project, group, event_id):
    event = get_object_or_404(group.event_set, pk=event_id)

    return render_to_response('sentry/groups/event.html', {
        'project': project,
        'page': 'event_list',
        'group': group,
        'event': event,
        'interface_list': _get_rendered_interfaces(event),
        'json_data': event.data.get('extra', {}),
        'version_data': event.data.get('modules', None),
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_access(MEMBER_USER)
def group_event_details_json(request, project, group_id, event_id_or_latest):
    group = get_object_or_404(Group, pk=group_id, project=project)

    if event_id_or_latest == 'latest':
        # It's possible that a message would not be created under certain
        # circumstances (such as a post_save signal failing)
        event = group.get_latest_event() or Event()
    else:
        event = get_object_or_404(group.event_set, pk=event_id_or_latest)

    return HttpResponse(json.dumps(event.as_dict()), mimetype='application/json')

@login_required
@has_access(MEMBER_USER)
def group_plugin_action(request, project, group_id, slug):
    group = get_object_or_404(Group, pk=group_id, project=project)

    try:
        plugin = plugins.get(slug)
    except KeyError:
        raise Http404('Plugin not found')

    response = plugin.get_view_response(request, group)
    if response:
        return response
    return HttpResponseRedirect(request.META.get('HTTP_REFERER') or reverse('sentry', kwargs={'project_id': group.project.slug}))
<commit_msg>Improved the performance and simplified the code for date filters on the error list page.<commit_after>"""
sentry.web.frontend.groups
~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

import datetime
import logging
import re

from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.safestring import mark_safe

from sentry.conf import settings
from sentry.constants import (SORT_OPTIONS, SEARCH_SORT_OPTIONS,
    SORT_CLAUSES, MYSQL_SORT_CLAUSES, SQLITE_SORT_CLAUSES, MEMBER_USER,
    SCORE_CLAUSES, MYSQL_SCORE_CLAUSES, SQLITE_SCORE_CLAUSES)
from sentry.filters import get_filters
from sentry.models import Group, Event, View, SearchDocument
from sentry.permissions import can_admin_group
from sentry.plugins import plugins
from sentry.utils import json
from sentry.utils.dates import parse_date
from sentry.utils.db import has_trending, get_db_engine
from sentry.web.decorators import has_access, has_group_access, login_required
from sentry.web.helpers import render_to_response

uuid_re = re.compile(r'^[a-z0-9]{32}$', re.I)
event_re = re.compile(r'^(?P<event_id>[a-z0-9]{32})\$(?P<checksum>[a-z0-9]{32})$', re.I)

def _get_rendered_interfaces(event):
    interface_list = []
    for interface in event.interfaces.itervalues():
        try:
            html = interface.to_html(event)
        except Exception:
            logger = logging.getLogger('sentry.interfaces')
            logger.exception('Error rendering interface %r', interface.__class__, extra={
                'event_id': event.id,
            })
            continue
        if not html:
            continue
        interface_list.append(mark_safe(html))
    return interface_list

def _get_group_list(request, project, view=None):
    filters = []
    for cls in get_filters(Group, project):
        try:
            filters.append(cls(request, project))
        except Exception, e:
            logger = logging.getLogger('sentry.filters')
            logger.exception('Error initializing filter %r: %s', cls, e)

    event_list = Group.objects
    if request.user.is_authenticated() and request.GET.get('bookmarks'):
        event_list = event_list.filter(
            bookmark_set__project=project,
            bookmark_set__user=request.user,
        )
    else:
        event_list = event_list.filter(project=project)

    if view:
        event_list = event_list.filter(views=view)

    for filter_ in filters:
        try:
            if not filter_.is_set():
                continue
            event_list = filter_.get_query_set(event_list)
        except Exception, e:
            logger = logging.getLogger('sentry.filters')
            logger.exception('Error processing filter %r: %s', cls, e)

    date_from = request.GET.get('df')
    time_from = request.GET.get('tf')
    date_to = request.GET.get('dt')
    time_to = request.GET.get('tt')

    today = timezone.now()

    # date format is Y-m-d
    if any(x is not None for x in [date_from, time_from, date_to, time_to]):
        date_from, date_to = parse_date(date_from, time_from), parse_date(date_to, time_to)
    else:
        date_from = today - datetime.timedelta(days=3)
        date_to = None

    if date_from and date_to:
        event_list = event_list.filter(
            messagecountbyminute__date__gte=date_from,
            messagecountbyminute__date__lte=date_to,
        )
    elif date_from:
        event_list = event_list.filter(last_seen__gte=date_from)
    elif date_to:
        event_list = event_list.filter(last_seen__lte=date_to)

    sort = request.GET.get('sort')
    if sort not in SORT_OPTIONS:
        sort = settings.DEFAULT_SORT_OPTION

    if sort.startswith('accel_') and not has_trending():
        sort = settings.DEFAULT_SORT_OPTION

    engine = get_db_engine('default')
    if engine.startswith('sqlite'):
        score_clause = SQLITE_SORT_CLAUSES.get(sort)
        filter_clause = SQLITE_SCORE_CLAUSES.get(sort)
    elif engine.startswith('mysql'):
        score_clause = MYSQL_SORT_CLAUSES.get(sort)
        filter_clause = MYSQL_SCORE_CLAUSES.get(sort)
    else:
        score_clause = SORT_CLAUSES.get(sort)
        filter_clause = SCORE_CLAUSES.get(sort)

    # All filters must already be applied once we reach this point
    if sort == 'tottime':
        event_list = event_list.filter(time_spent_count__gt=0)
    elif sort == 'avgtime':
        event_list = event_list.filter(time_spent_count__gt=0)
    elif sort.startswith('accel_'):
        event_list = Group.objects.get_accelerated(event_list, minutes=int(sort.split('_', 1)[1]))

    if score_clause:
        event_list = event_list.extra(
            select={'sort_value': score_clause},
        ).order_by('-sort_value', '-last_seen')
        cursor = request.GET.get('cursor')
        if cursor:
            event_list = event_list.extra(
                where=['%s > %%s' % filter_clause],
                params=[cursor],
            )

    return {
        'filters': filters,
        'event_list': event_list,
        'date_from': date_from,
        'date_to': date_to,
        'today': today,
        'sort': sort,
    }

@login_required
@has_access
def search(request, project):
    query = request.GET.get('q')

    if not query:
        return HttpResponseRedirect(reverse('sentry', args=[project.slug]))

    sort = request.GET.get('sort')
    if sort not in SEARCH_SORT_OPTIONS:
        sort = settings.SEARCH_DEFAULT_SORT_OPTION
    sort_label = SEARCH_SORT_OPTIONS[sort]

    result = event_re.match(query)
    if result:
        # Forward to message if it exists
        # event_id = result.group(1)
        checksum = result.group(2)
        event_list = Group.objects.filter(checksum=checksum)
        top_matches = event_list[:2]
        if len(top_matches) == 0:
            return render_to_response('sentry/invalid_message_id.html', {
                'project': project,
            }, request)
        elif len(top_matches) == 1:
            return HttpResponseRedirect(top_matches[0].get_absolute_url())
    elif uuid_re.match(query):
        # Forward to message if it exists
        try:
            message = Event.objects.get(project=project, event_id=query)
        except Event.DoesNotExist:
            return render_to_response('sentry/invalid_message_id.html', {
                'project': project,
            }, request)
        else:
            return HttpResponseRedirect(message.get_absolute_url())
    elif not settings.USE_SEARCH:
        event_list = Group.objects.none()
        # return render_to_response('sentry/invalid_message_id.html', {
        #         'project': project,
        #     }, request)
    else:
        documents = list(SearchDocument.objects.search(project, query, sort_by=sort))
        groups = Group.objects.in_bulk([d.group_id for d in documents])

        event_list = []
        for doc in documents:
            try:
                event_list.append(groups[doc.group_id])
            except KeyError:
                continue

    return render_to_response('sentry/search.html', {
        'project': project,
        'event_list': event_list,
        'query': query,
        'sort': sort,
        'sort_label': sort_label,
    }, request)

@login_required
@has_access
def group_list(request, project):
    try:
        page = int(request.GET.get('p', 1))
    except (TypeError, ValueError):
        page = 1

    view_id = request.GET.get('view')
    if view_id:
        try:
            view = View.objects.get_from_cache(pk=int(view_id))
        except View.DoesNotExist:
            return HttpResponseRedirect(reverse('sentry', args=[project.slug]))
    else:
        view = None

    response = _get_group_list(
        request=request,
        project=project,
        view=view,
    )

    # XXX: this is duplicate in _get_group_list
    sort_label = SORT_OPTIONS[response['sort']]

    has_realtime = page == 1

    return render_to_response('sentry/groups/group_list.html', {
        'project': project,
        'from_date': response['date_from'],
        'to_date': response['date_to'],
        'has_realtime': has_realtime,
        'event_list': response['event_list'],
        'today': response['today'],
        'sort': response['sort'],
        'sort_label': sort_label,
        'filters': response['filters'],
        'view': view,
        'SORT_OPTIONS': SORT_OPTIONS,
        'HAS_TRENDING': has_trending(),
        'PAGE': 'dashboard',
    }, request)

@has_group_access
def group(request, project, group):
    # It's possible that a message would not be created under certain
    # circumstances (such as a post_save signal failing)
    event = group.get_latest_event() or Event(group=group)

    return render_to_response('sentry/groups/details.html', {
        'project': project,
        'page': 'details',
        'group': group,
        'event': event,
        'interface_list': _get_rendered_interfaces(event),
        'json_data': event.data.get('extra', {}),
        'version_data': event.data.get('modules', None),
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_group_access
def group_tag_details(request, project, group, tag_name):
    return render_to_response('sentry/plugins/bases/tag/index.html', {
        'project': project,
        'group': group,
        'title': tag_name.replace('_', ' ').title(),
        'tag_name': tag_name,
        'unique_tags': group.get_unique_tags(tag_name),
        'group': group,
        'page': 'tag_details',
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_group_access
def group_event_list(request, project, group):
    event_list = group.event_set.all().order_by('-datetime')

    return render_to_response('sentry/groups/event_list.html', {
        'project': project,
        'group': group,
        'event_list': event_list,
        'page': 'event_list',
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_access(MEMBER_USER)
def group_event_list_json(request, project, group_id):
    group = get_object_or_404(Group, pk=group_id, project=project)

    limit = request.GET.get('limit', settings.MAX_JSON_RESULTS)
    try:
        limit = int(limit)
    except ValueError:
        return HttpResponse('non numeric limit', status=400, mimetype='text/plain')
    if limit > settings.MAX_JSON_RESULTS:
        return HttpResponse("too many objects requested", mimetype='text/plain', status=400)

    events = group.event_set.order_by('-id')[:limit]

    return HttpResponse(json.dumps([event.as_dict() for event in events]), mimetype='application/json')

@has_group_access
def group_event_details(request, project, group, event_id):
    event = get_object_or_404(group.event_set, pk=event_id)

    return render_to_response('sentry/groups/event.html', {
        'project': project,
        'page': 'event_list',
        'group': group,
        'event': event,
        'interface_list': _get_rendered_interfaces(event),
        'json_data': event.data.get('extra', {}),
        'version_data': event.data.get('modules', None),
        'can_admin_event': can_admin_group(request.user, group),
    }, request)

@has_access(MEMBER_USER)
def group_event_details_json(request, project, group_id, event_id_or_latest):
    group = get_object_or_404(Group, pk=group_id, project=project)

    if event_id_or_latest == 'latest':
        # It's possible that a message would not be created under certain
        # circumstances (such as a post_save signal failing)
        event = group.get_latest_event() or Event()
    else:
        event = get_object_or_404(group.event_set, pk=event_id_or_latest)

    return HttpResponse(json.dumps(event.as_dict()), mimetype='application/json')

@login_required
@has_access(MEMBER_USER)
def group_plugin_action(request, project, group_id, slug):
    group = get_object_or_404(Group, pk=group_id, project=project)

    try:
        plugin = plugins.get(slug)
    except KeyError:
        raise Http404('Plugin not found')

    response = plugin.get_view_response(request, group)
    if response:
        return response
    return HttpResponseRedirect(request.META.get('HTTP_REFERER') or reverse('sentry', kwargs={'project_id': group.project.slug}))
<|endoftext|>


<commit_before># test1.py
# Ronald L. Rivest and Karim Husayn Karimi
# August 17, 2017
# Routine to experiment with scipy.optimize.minimize

import scipy.optimize

# function to minimize:
def g(xy):
    (x,y) = xy
    return x+y

# constraints

# constraint 1: y <= x/2
def f1(xy):
    (x,y) = xy
    return x/2 - y

# constraint 2: y >= 1/x
def f2(xy):
    (x,y) = xy
    return y - 1.0/x

constraints = [
                { "type": "ineq",
                  "fun": f1
                },
                { "type": "ineq",
                  "fun": f2
                }
               ]

print(scipy.optimize.minimize(g, (10, 5), constraints=constraints))

<commit_msg>Add noise to opt/test1.py example.<commit_after># test1.py
# Ronald L. Rivest and Karim Husayn Karimi
# August 17, 2017
# Routine to experiment with scipy.optimize.minimize

import scipy.optimize
from scipy.stats import norm

# function to minimize:
def g(xy):
    (x,y) = xy
    print("g({},{})".format(x,y))
    return x + y + 0.00000005 * norm.rvs(0)

# constraints

# constraint 1: y <= x/2
def f1(xy):
    (x,y) = xy
    return x/2 - y

# constraint 2: y >= 1/x
def f2(xy):
    (x,y) = xy
    return y - 1.0/x

constraints = [
                { "type": "ineq",
                  "fun": f1
                },
                { "type": "ineq",
                  "fun": f2
                }
               ]

print(scipy.optimize.minimize(g, (11, 5), constraints=constraints))

<|endoftext|>


<commit_before><commit_msg>TEST: register_multimodal: added test for slicewise registration<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
datastore.py

Created by Chris Mulligan on 2010-05-09.
Copyright (c) 2010 __MyCompanyName__. All rights reserved.
"""
import os

try:
    import simplejson as json
except ImportError:
    import json

import uuid
from string import letters, digits
from random import choice

def shortkey():
    firstlast = list(letters + digits)
    middle = firstlast + list('-_.')
    return ''.join((choice(firstlast), choice(middle), choice(middle), choice(middle), choice(firstlast)))

class DataStore(object):
    """Implements a datastore using an arbitrary backend. """
    def __init__(self, *args, **kwargs):
        pass

    def _store(self, uid, content):
        """Store the given dict of content at uid. Nothing returned."""
        raise NotImplementedError

    def _storeLog(self, nick, time, uid):
        """Adds the nick & uid to the log for a given time/order. No return."""
        raise NotImplementedError

    def _retrieve(self, uid):
        """Return a dict with the contents of the paste, including the raw
        data, if any, as the key 'data'. Must pass in uid, not shortid."""
        raise NotImplementedError

    def lookup(self, nick):
        """Looks for the most recent paste by a given nick.
        Returns the uid or None"""
        raise NotImplementedError

    def store(self, type, nick, time, fmt=None, code=None, filename=None, mime=None, data=None, makeshort=True):
        """Store code or a file. Returns a tuple containing the uid and shortid"""
        uid = str(uuid.uuid4())
        if makeshort:
            shortid = shortkey()
        else:
            shortid = None

        paste = {'uid' : uid, 'shortid' : shortid, 'type' : type, 'nick' : nick, 'time' : time,
            'fmt' : fmt, 'code' : code,
            'filename' : filename, 'mime' : mime, 'data' : data}
        self._store(uid, paste)
        if nick:
            self._storeLog(nick, time, uid)
        return (uid, shortid)

    def retrieve(self, id):
        """Retrieve a paste. Returns a dictionary containing all metadata
        and the file data, if it's a file."""
        if len(id) < 10:
            shortid = id
            uid = self._lookupUid(shortid)
        else:
            uid = id
        return self._retrieve(uid)

class JsonDataStore(DataStore):
    """Stores using json encoded into files"""
    def __init__(self, repo):
        super(JsonDataStore, self).__init__()
        self.repo = repo
        if not os.path.exists(repo):
            os.mkdirs(repo)
        self.shortids = {}

    def _store(self, uid, content):
        data = content.pop('data')
        fd = open(os.path.join(self.repo, uid), 'wb')
        fd.write(json.dumps(content))
        fd.close()
        if data:
            fd = open(os.path.join(self.repo, '%s.raw' % uid), 'wb')
            fd.write(data)
            fd.close()
        if content['shortid']:
            self.shortids[content['shortid']] = uid
            open(os.path.join(self.repo, 'shortids.txt'), 'a').write('%s %s\n' % (content['shortid'], uid))

    def _storeLog(self, nick, time, uid):
        open(os.path.join(self.repo, 'log.txt'), 'a').write('%s %s\n' % (nick, uid))

    def lookup(self, nick):
        last = None
        for line in open(os.path.join(self.repo, 'log.txt')):
            who, uid = line.strip().rsplit(None, 1)
            if who == nick:
                last = uid
        return last

    def _lookupUid(self, shortid):
        if not self.shortids:
            for line in open(os.path.join(self.repo, 'shortids.txt')):
                l1, l2 = line.strip().rsplit(None, 1)
                self.shortids[l1] = l2
        try:
            uid = self.shortids[shortid]
        except KeyError:
            uid = None
        return uid

    def _retrieve(self, uid):
        paste = json.loads(open(os.path.join(self.repo, uid), 'rb').read())
        if paste['type'] == 'file':
            paste['data'] = open(os.path.join(self.repo, '%s.raw' % uid), 'rb').read()
        return paste<commit_msg>Filter list<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
datastore.py

Created by Chris Mulligan on 2010-05-09.
Copyright (c) 2010 __MyCompanyName__. All rights reserved.
"""
import os

try:
    import simplejson as json
except ImportError:
    import json

import uuid
from string import letters, digits
from random import choice

def shortkey():
    firstlast = list(letters + digits)
    middle = firstlast + list('-_.')
    return ''.join((choice(firstlast), choice(middle), choice(middle), choice(middle), choice(firstlast)))

class DataStore(object):
    """Implements a datastore using an arbitrary backend. """
    def __init__(self, *args, **kwargs):
        pass

    def _store(self, uid, content):
        """Store the given dict of content at uid. Nothing returned."""
        raise NotImplementedError

    def _storeLog(self, nick, time, uid):
        """Adds the nick & uid to the log for a given time/order. No return."""
        raise NotImplementedError

    def _retrieve(self, uid):
        """Return a dict with the contents of the paste, including the raw
        data, if any, as the key 'data'. Must pass in uid, not shortid."""
        raise NotImplementedError

    def lookup(self, nick):
        """Looks for the most recent paste by a given nick.
        Returns the uid or None"""
        raise NotImplementedError

    def store(self, type, nick, time, fmt=None, code=None, filename=None, mime=None, data=None, makeshort=True):
        """Store code or a file. Returns a tuple containing the uid and shortid"""
        uid = str(uuid.uuid4())
        if makeshort:
            shortid = shortkey()
        else:
            shortid = None

        temp = {'uid' : uid, 'shortid' : shortid, 'type' : type, 'nick' : nick, 'time' : time,
            'fmt' : fmt, 'code' : code,
            'filename' : filename, 'mime' : mime, 'data' : data}
        paste = {}
        for k, v in temp.items():
            if v:
                paste[k] = v
        self._store(uid, paste)
        if nick:
            self._storeLog(nick, time, uid)
        return (uid, shortid)

    def retrieve(self, id):
        """Retrieve a paste. Returns a dictionary containing all metadata
        and the file data, if it's a file."""
        if len(id) < 10:
            shortid = id
            uid = self._lookupUid(shortid)
        else:
            uid = id
        return self._retrieve(uid)

class JsonDataStore(DataStore):
    """Stores using json encoded into files"""
    def __init__(self, repo):
        super(JsonDataStore, self).__init__()
        self.repo = repo
        if not os.path.exists(repo):
            os.mkdirs(repo)
        self.shortids = {}

    def _store(self, uid, content):
        data = content.pop('data')
        fd = open(os.path.join(self.repo, uid), 'wb')
        fd.write(json.dumps(content))
        fd.close()
        if data:
            fd = open(os.path.join(self.repo, '%s.raw' % uid), 'wb')
            fd.write(data)
            fd.close()
        if content['shortid']:
            self.shortids[content['shortid']] = uid
            open(os.path.join(self.repo, 'shortids.txt'), 'a').write('%s %s\n' % (content['shortid'], uid))

    def _storeLog(self, nick, time, uid):
        open(os.path.join(self.repo, 'log.txt'), 'a').write('%s %s\n' % (nick, uid))

    def lookup(self, nick):
        last = None
        for line in open(os.path.join(self.repo, 'log.txt')):
            who, uid = line.strip().rsplit(None, 1)
            if who == nick:
                last = uid
        return last

    def _lookupUid(self, shortid):
        if not self.shortids:
            for line in open(os.path.join(self.repo, 'shortids.txt')):
                l1, l2 = line.strip().rsplit(None, 1)
                self.shortids[l1] = l2
        try:
            uid = self.shortids[shortid]
        except KeyError:
            uid = None
        return uid

    def _retrieve(self, uid):
        paste = json.loads(open(os.path.join(self.repo, uid), 'rb').read())
        if paste['type'] == 'file':
            paste['data'] = open(os.path.join(self.repo, '%s.raw' % uid), 'rb').read()
        return paste<|endoftext|>


<commit_before>import os
import datetime
import tornado.gen
import statsd

if os.environ.get("STATSD_URL", None):
    ip, port = os.environ.get("STATSD_URL").split(':')
    statsd_client = statsd.Client(ip, int(port))
else:
    statsd_client = None

def instrument(name):
    def instrument_wrapper(fn):
        @tornado.gen.coroutine
        def fn_wrapper(*args, **kwargs):
            def send_duration():
                duration = (datetime.datetime.now() - start).total_seconds()
                statsd_client.timing("{}.duration".format(name),
                                     int(duration * 1000))

            start = datetime.datetime.now()
            ftr_result = fn(*args, **kwargs)

            try:
                result = yield tornado.gen.maybe_future(ftr_result)
            except Exception as e:
                if statsd_client:
                    statsd_client.incr("{}.exceptions.{}".format(
                        name, e.__class__.__name__.lower()))
                    send_duration()

                raise e
            else:
                if statsd_client:
                    send_duration()
                    statsd_client.incr("{}.success".format(name))

                raise tornado.gen.Return(result)

        return fn_wrapper
    return instrument_wrapper
<commit_msg>Update statsd connector<commit_after>import os
import datetime
import tornado.gen
import statsd

if os.environ.get("STATSD_URL", None):
    ip, port = os.environ.get("STATSD_URL").split(':')
    statsd_client = statsd.StatsClient(host=ip, port=int(port),
                                       prefix=os.environ.get('STATSD_PREFIX',
                                                             None))
else:
    statsd_client = None

def instrument(name):
    def instrument_wrapper(fn):
        @tornado.gen.coroutine
        def fn_wrapper(*args, **kwargs):
            def send_duration():
                duration = (datetime.datetime.now() - start).total_seconds()
                statsd_client.timing("{}.duration".format(name),
                                     int(duration * 1000))

            start = datetime.datetime.now()
            ftr_result = fn(*args, **kwargs)

            try:
                result = yield tornado.gen.maybe_future(ftr_result)
            except Exception as e:
                if statsd_client:
                    statsd_client.incr("{}.exceptions.{}".format(
                        name, e.__class__.__name__.lower()))
                    send_duration()

                raise e
            else:
                if statsd_client:
                    send_duration()
                    statsd_client.incr("{}.success".format(name))

                raise tornado.gen.Return(result)

        return fn_wrapper
    return instrument_wrapper
<|endoftext|>


<commit_before>import json
from pyramid.response import Response
from urllib2 import Request, urlopen, URLError, HTTPError
import os

# http://0.0.0.0:6545/run_gotools?aspect=C&genes=COR5|CYT1|Q0105|QCR2|S000001929|S000002937|S000003809|YEL024W|YEL039C|YGR183C|YHR001W-A

gotermfinder_url = "http://gotermfinder.yeastgenome.org/cgi-bin/gotermfinder2"

goslimmapper_url = "http://gotermfinder.yeastgenome.org/cgi-bin/goslimmapper2"

# http://gotermfinder.yeastgenome.org/cgi-bin/gotermfinder?aspect=F&genes=COR5|CYT1|Q0105|QCR2|S000001929|S000002937|S000003809|YEL024W|YEL039C|YGR183C|YHR001W-A

def do_gosearch(request):

    p = dict(request.params)

    data = {}
    if p.get('mapper'):
        data = run_goslimmapper(p)
    else:
        data = run_gotermfinder(p)
    return Response(body=json.dumps(data), content_type='application/json')

def run_goslimmapper(p):

    genes = p.get('genes', '')
    if genes == '':
        return { " ERROR": "NO GENE NAME PASSED IN" }
    aspect = p.get('aspect', '')
    if aspect == '':
        return { " ERROR": "NO GO ASPECT PASSED IN" }
    terms = p.get('terms', '')
    if terms == '':
        return { " ERROR": "NO SLIM TERMS PASSED IN" }

    import urllib

    paramData = urllib.urlencode({ 'genes': genes,
                                   'terms': terms,
                                   'aspect': aspect });

    res = _get_json_from_server(goslimmapper_url, paramData)

    if res.get('output'):
        return { "output": res['output'] }

    htmlUrl = res['html']
    rootUrl = res['rootUrl']

    response = urlopen(htmlUrl)
    html = response.read()
    html = html.replace("", "").replace("", "")
    html = html.replace("
", "").replace("

", "")
    html = html.replace("color=red", "color=maroon")
    html = html.replace("", "")
    html = html.replace("", "").replace("", "")
    html = html.replace("", "")
    html = html.replace("", "")
    html = html.replace('
', '')
    html = html.replace(' nowrap=""', '')
    html = html.replace('( ', '(')
    html = html.replace(' )', ')')

    # html = html.replace("http://amigo.geneontology.org/amigo/term/", "https://www.yeastgenome.org/go/")
    html = html.replace("infowin", "_extwin")

    return { "html": html,
             "tab_page": res['tab'],
             "term_page": res['terms'],
             "table_page": res['html'],
             "gene_input_page": res['gene_input'],
             "slim_input_page": res['slim_input'] }

def run_gotermfinder(p):

    genes = p.get('genes', '')
    aspect = p.get('aspect', 'F')
    if genes == '':
        return { " ERROR": "NO GENE NAME PASSED IN" }
    genes4bg = p.get('genes4bg', '')
    pvalue = p.get('pvalue', '0.01')
    FDR = p.get('FDR', 0)
    evidence = p.get('evidence', '')

    ## add code later to handle pvalue + exclude evidence list etc
    # url = gotermfinder_url + "?aspect=" + aspect + "&genes=" + genes

    import urllib

    paramData = urllib.urlencode({ 'genes': genes,
                                   'genes4bg': genes4bg,
                                   'aspect': aspect,
                                   'pvalue': pvalue,
                                   'FDR': FDR,
                                   'evidence': evidence });

    res = _get_json_from_server(gotermfinder_url, paramData)

    if res.get('output'):
        return { "output": res['output'] }

    htmlUrl = res['html']
    rootUrl = res['rootUrl']
    imageHtmlUrl = res['imageHtml']

    response = urlopen(imageHtmlUrl)
    imageHtml = response.read()
    imageHtml = imageHtml.replace("", "").replace("", "")
    imageHtml = imageHtml.replace("<img src='./", "<img src='" + rootUrl + "/")
    imageHtml = "Nodes" + imageHtml.split("

Nodes")[1]

    response = urlopen(htmlUrl)
    html = response.read()
    html = html.replace("", "").replace("", "")
    html = html.replace("color=red", "color=maroon")
    html = html.replace('', '')
    html = html.replace("infowin", "_extwin")

    return { "html": html,
             "image_html": imageHtml,
             "tab_page": res['tab'],
             "term_page": res['terms'],
             "image_page": res['imageHtml'],
             "table_page": res['html'],
             "svg_page": res['svg'],
             "png_page": res['png'],
             "ps_page": res['ps'],
             "input_page": res['input'] }

def _get_json_from_server(url, paramData):

    try:
        req = Request(url=url, data=paramData)
        res = urlopen(req)
        data = json.loads(res.read())
        return data
    except HTTPError:
        return 404

<commit_msg>fixed to use the current TermFinder/mapper program<commit_after>import json
from pyramid.response import Response
from urllib2 import Request, urlopen, URLError, HTTPError
import os

# http://0.0.0.0:6545/run_gotools?aspect=C&genes=COR5|CYT1|Q0105|QCR2|S000001929|S000002937|S000003809|YEL024W|YEL039C|YGR183C|YHR001W-A

gotermfinder_url = "http://gotermfinder.yeastgenome.org/cgi-bin/gotermfinder"

goslimmapper_url = "http://gotermfinder.yeastgenome.org/cgi-bin/goslimmapper"

# http://gotermfinder.yeastgenome.org/cgi-bin/gotermfinder?aspect=F&genes=COR5|CYT1|Q0105|QCR2|S000001929|S000002937|S000003809|YEL024W|YEL039C|YGR183C|YHR001W-A

def do_gosearch(request):

    p = dict(request.params)

    data = {}
    if p.get('mapper'):
        data = run_goslimmapper(p)
    else:
        data = run_gotermfinder(p)
    return Response(body=json.dumps(data), content_type='application/json')

def run_goslimmapper(p):

    genes = p.get('genes', '')
    if genes == '':
        return { " ERROR": "NO GENE NAME PASSED IN" }
    aspect = p.get('aspect', '')
    if aspect == '':
        return { " ERROR": "NO GO ASPECT PASSED IN" }
    terms = p.get('terms', '')
    if terms == '':
        return { " ERROR": "NO SLIM TERMS PASSED IN" }

    import urllib

    paramData = urllib.urlencode({ 'genes': genes,
                                   'terms': terms,
                                   'aspect': aspect });

    res = _get_json_from_server(goslimmapper_url, paramData)

    if res.get('output'):
        return { "output": res['output'] }

    htmlUrl = res['html']
    rootUrl = res['rootUrl']

    response = urlopen(htmlUrl)
    html = response.read()
    html = html.replace("", "").replace("", "")
    html = html.replace("
", "").replace("

", "")
    html = html.replace("color=red", "color=maroon")
    html = html.replace("", "")
    html = html.replace("", "").replace("", "")
    html = html.replace("", "")
    html = html.replace("", "")
    html = html.replace('
', '')
    html = html.replace(' nowrap=""', '')
    html = html.replace('( ', '(')
    html = html.replace(' )', ')')

    # html = html.replace("http://amigo.geneontology.org/amigo/term/", "https://www.yeastgenome.org/go/")
    html = html.replace("infowin", "_extwin")

    return { "html": html,
             "tab_page": res['tab'],
             "term_page": res['terms'],
             "table_page": res['html'],
             "gene_input_page": res['gene_input'],
             "slim_input_page": res['slim_input'] }

def run_gotermfinder(p):

    genes = p.get('genes', '')
    aspect = p.get('aspect', 'F')
    if genes == '':
        return { " ERROR": "NO GENE NAME PASSED IN" }
    genes4bg = p.get('genes4bg', '')
    pvalue = p.get('pvalue', '0.01')
    FDR = p.get('FDR', 0)
    evidence = p.get('evidence', '')

    ## add code later to handle pvalue + exclude evidence list etc
    # url = gotermfinder_url + "?aspect=" + aspect + "&genes=" + genes

    import urllib

    paramData = urllib.urlencode({ 'genes': genes,
                                   'genes4bg': genes4bg,
                                   'aspect': aspect,
                                   'pvalue': pvalue,
                                   'FDR': FDR,
                                   'evidence': evidence });

    res = _get_json_from_server(gotermfinder_url, paramData)

    if res.get('output'):
        return { "output": res['output'] }

    htmlUrl = res['html']
    rootUrl = res['rootUrl']
    imageHtmlUrl = res['imageHtml']

    response = urlopen(imageHtmlUrl)
    imageHtml = response.read()
    imageHtml = imageHtml.replace("", "").replace("", "")
    imageHtml = imageHtml.replace("<img src='./", "<img src='" + rootUrl + "/")
    imageHtml = "Nodes" + imageHtml.split("

Nodes")[1]

    response = urlopen(htmlUrl)
    html = response.read()
    html = html.replace("", "").replace("", "")
    html = html.replace("color=red", "color=maroon")
    html = html.replace('', '')
    html = html.replace("infowin", "_extwin")

    return { "html": html,
             "image_html": imageHtml,
             "tab_page": res['tab'],
             "term_page": res['terms'],
             "image_page": res['imageHtml'],
             "table_page": res['html'],
             "svg_page": res['svg'],
             "png_page": res['png'],
             "ps_page": res['ps'],
             "input_page": res['input'] }

def _get_json_from_server(url, paramData):

    try:
        req = Request(url=url, data=paramData)
        res = urlopen(req)
        data = json.loads(res.read())
        return data
    except HTTPError:
        return 404

<|endoftext|>