<commit_before># -*- coding: utf-8 -*-
from django import forms
from django.db.models import Q
from django.views.generic import ListView
from django.views.generic.edit import FormMixin
from django.utils.translation import ugettext_lazy as _

from bunch import Bunch

def is_filter(value, form):
    return bool(value in form.cleaned_data and form.cleaned_data[value])

class FilteredListView(FormMixin, ListView):
    """A Generic ListView used to filter and order objects."""
    default_order = None

    def is_form_submitted(self):
        """
        Return True if the form is already submited. False otherwise
        """
        return bool(self.request.method == 'GET' and self.request.GET)

    def get_initial(self):
        """
        add "order_by" and "order_reverse" to the initials.
        """
        kwargs = super(FilteredListView, self).get_initial()
        if self.default_order:
            order_by = self.default_order

            kwargs['order_by'] = order_by
            kwargs['order_reverse'] = order_by.startswith('-')

        return kwargs

    def get_form_kwargs(self):
        """Read GET data to return keyword arguments for the form."""
        kwargs = {'initial': self.get_initial()}
        if self.is_form_submitted():
            kwargs.update({'data': self.request.GET})
        return kwargs

    def __get_queryset(self):
        """Helper to get ListView default queryset."""
        return super(ListView, self).get_queryset()

    def get_queryset(self):
        """Return filtered queryset. Uses form_valid() or form_invalid()."""
        if self.form.is_valid():
            return self.form_valid(self.form)
        else:
            return self.form_invalid(self.form)

    def get_qs_filters(self):
        """
        retreive filters from "qs_filter_fields" or "filter_fields"
        and return them as a dict to be used by self.form_valid
        """

        filters = {}

        if hasattr(self, 'qs_filter_fields'):
            filters = self.qs_filter_fields

        elif hasattr(self, 'filter_fields'):
            for field in self.filter_fields:
                filters[field] = field

        return filters

    def form_valid(self, form):
        """
        The form_valid is reponsible for filtering and ordering the
        base queryset. It return a queryset.

        :param: `django.forms.Forms`
        :return: `django.db.models.query.QuerySet`
        """
        # Get default queryset from ListView parameters (queryset, model, ...)
        queryset = self.__get_queryset()

        # Handle QueryFormMixin
        if is_filter('query', form):
            query = form.cleaned_data['query']
            query_words = query.split()
            filters = None
            for f in self.search_fields:
                for word in query_words:
                    q = Q(**{f + '__icontains': word})
                    filters = filters | q if filters else q
            if filters:
                queryset = queryset.filter(filters)

        # Handle get_qs_filters
        filters = {}
        for k, v in self.get_qs_filters().iteritems():
            field = form.cleaned_data.get(v)
            if field:
                filters[k] = field

                # Get extra condition for a field to on the filters
                if hasattr(self, 'qs_filter_fields_conditions'):
                    try:
                        filter_fields_condtion = self.\
                            qs_filter_fields_conditions[k]
                    except KeyError:
                        filter_fields_condtion = {}

                    for key, value in filter_fields_condtion.iteritems():
                        filters[key] = value

        queryset = queryset.filter(**filters)

        # Handle OrderFormMixin
        if is_filter('order_by', form):
            order_field = form.cleaned_data['order_by']
            queryset = queryset.order_by(order_field)

        if is_filter('order_reverse', form):
            queryset = queryset.reverse()

        # Handle distinct for Join in some Querysets
        queryset = queryset.distinct()

        return queryset

    def form_invalid(self, form):
        """Return queryset when submitted form is invalid.

        Default implementation uses :meth:`form_empty`.

        :param: `django.forms.Forms`
        :return: `django.db.models.query.QuerySet`

        """
        return self.form_empty()

    def form_empty(self):
        """Return queryset used when form is not submitted."""
        queryset = self.__get_queryset()
        if self.default_order:
            queryset = queryset.order_by(self.default_order)

        return queryset

    @property
    def form(self):
        """
        guess the form to be used and hide the filter_fields in the template

        """
        try:
            return self._form
        except AttributeError:
            form_class = self.get_form_class()
            self._form = self.get_form(form_class)

            # Hide filter_fields
            if hasattr(self, 'filter_fields'):
                for field in self.filter_fields:
                    self._form.fields[field].widget = forms.HiddenInput()

            return self._form

    def get_context_data(self, **kwargs):
        """
        Add a list of filters and self.form to the context to be rendered by
        the view.
        """
        kwargs = ListView.get_context_data(self, **kwargs)
        kwargs['form'] = self.form
        kwargs['filters'] = self.get_filters()

        return kwargs

    def get_filters(self):
        """
        Convert some ChoiceField in a list of choices in the
        template.
        """
        filters = []

        if hasattr(self, 'filter_fields'):
            for field in self.filter_fields:
                new_filter = Bunch()
                new_filter.label = self.form.fields[field].label
                new_filter.name = field
                new_filter.choices = []
                selected = False
                for choice in self.form.fields[field].choices:
                    new_choice = Bunch()
                    new_choice.value = choice[0]
                    new_choice.label = choice[1]
                    if hasattr(self.form, 'cleaned_data') and \
                            field in self.form.cleaned_data:
                        # Get value for ModelChoiceField or ChoiceField
                        value = getattr(self.form.cleaned_data[field], 'pk',
                                        self.form.cleaned_data[field])
                        if value == choice[0]:
                            new_choice.is_selected = True
                            selected = True
                        else:
                            new_choice.is_selected = False
                    else:
                        new_choice.is_selected = False
                    new_filter.choices.append(new_choice)

                if not self.form.fields[field].required and \
                        not [c for c in new_filter.choices if c.value == '']:
                    all_choice = Bunch(value='', label=_('All'),
                                       is_selected=(not selected))
                    new_filter.choices.insert(0, all_choice)
                filters.append(new_filter)

        return filters
<commit_msg>[Controllers] Typo.<commit_after># -*- coding: utf-8 -*-
from django import forms
from django.db.models import Q
from django.views.generic import ListView
from django.views.generic.edit import FormMixin
from django.utils.translation import ugettext_lazy as _

from bunch import Bunch

def is_filter(value, form):
    return bool(value in form.cleaned_data and form.cleaned_data[value])

class FilteredListView(FormMixin, ListView):
    """A Generic ListView used to filter and order objects."""
    default_order = None

    def is_form_submitted(self):
        """
        Return True if the form is already submited. False otherwise
        """
        return bool(self.request.method == 'GET' and self.request.GET)

    def get_initial(self):
        """
        add "order_by" and "order_reverse" to the initials.
        """
        kwargs = super(FilteredListView, self).get_initial()
        if self.default_order:
            order_by = self.default_order

            kwargs['order_by'] = order_by
            kwargs['order_reverse'] = order_by.startswith('-')

        return kwargs

    def get_form_kwargs(self):
        """Read GET data to return keyword arguments for the form."""
        kwargs = {'initial': self.get_initial()}
        if self.is_form_submitted():
            kwargs.update({'data': self.request.GET})
        return kwargs

    def __get_queryset(self):
        """Helper to get ListView default queryset."""
        return super(ListView, self).get_queryset()

    def get_queryset(self):
        """Return filtered queryset. Uses form_valid() or form_invalid()."""
        if self.form.is_valid():
            return self.form_valid(self.form)
        else:
            return self.form_invalid(self.form)

    def get_qs_filters(self):
        """
        retreive filters from "qs_filter_fields" or "filter_fields"
        and return them as a dict to be used by self.form_valid
        """

        filters = {}

        if hasattr(self, 'qs_filter_fields'):
            filters = self.qs_filter_fields

        elif hasattr(self, 'filter_fields'):
            for field in self.filter_fields:
                filters[field] = field

        return filters

    def form_valid(self, form):
        """
        The form_valid is reponsible for filtering and ordering the
        base queryset. It return a queryset.

        :param: `django.forms.Forms`
        :return: `django.db.models.query.QuerySet`
        """
        # Get default queryset from ListView parameters (queryset, model, ...)
        queryset = self.__get_queryset()

        # Handle QueryFormMixin
        if is_filter('query', form):
            query = form.cleaned_data['query']
            query_words = query.split()
            filters = None
            for f in self.search_fields:
                for word in query_words:
                    q = Q(**{f + '__icontains': word})
                    filters = filters | q if filters else q
            if filters:
                queryset = queryset.filter(filters)

        # Handle get_qs_filters
        filters = {}
        for k, v in self.get_qs_filters().iteritems():
            field = form.cleaned_data.get(v)
            if field:
                filters[k] = field

                # Get extra condition for a field to on the filters
                if hasattr(self, 'qs_filter_fields_conditions'):
                    try:
                        filter_fields_conditions = self.\
                            qs_filter_fields_conditions[k]
                    except KeyError:
                        filter_fields_conditions = {}

                    for key, value in filter_fields_conditions.iteritems():
                        filters[key] = value

        queryset = queryset.filter(**filters)

        # Handle OrderFormMixin
        if is_filter('order_by', form):
            order_field = form.cleaned_data['order_by']
            queryset = queryset.order_by(order_field)

        if is_filter('order_reverse', form):
            queryset = queryset.reverse()

        # Handle distinct for Join in some Querysets
        queryset = queryset.distinct()

        return queryset

    def form_invalid(self, form):
        """Return queryset when submitted form is invalid.

        Default implementation uses :meth:`form_empty`.

        :param: `django.forms.Forms`
        :return: `django.db.models.query.QuerySet`

        """
        return self.form_empty()

    def form_empty(self):
        """Return queryset used when form is not submitted."""
        queryset = self.__get_queryset()
        if self.default_order:
            queryset = queryset.order_by(self.default_order)

        return queryset

    @property
    def form(self):
        """
        guess the form to be used and hide the filter_fields in the template

        """
        try:
            return self._form
        except AttributeError:
            form_class = self.get_form_class()
            self._form = self.get_form(form_class)

            # Hide filter_fields
            if hasattr(self, 'filter_fields'):
                for field in self.filter_fields:
                    self._form.fields[field].widget = forms.HiddenInput()

            return self._form

    def get_context_data(self, **kwargs):
        """
        Add a list of filters and self.form to the context to be rendered by
        the view.
        """
        kwargs = ListView.get_context_data(self, **kwargs)
        kwargs['form'] = self.form
        kwargs['filters'] = self.get_filters()

        return kwargs

    def get_filters(self):
        """
        Convert some ChoiceField in a list of choices in the
        template.
        """
        filters = []

        if hasattr(self, 'filter_fields'):
            for field in self.filter_fields:
                new_filter = Bunch()
                new_filter.label = self.form.fields[field].label
                new_filter.name = field
                new_filter.choices = []
                selected = False
                for choice in self.form.fields[field].choices:
                    new_choice = Bunch()
                    new_choice.value = choice[0]
                    new_choice.label = choice[1]
                    if hasattr(self.form, 'cleaned_data') and \
                            field in self.form.cleaned_data:
                        # Get value for ModelChoiceField or ChoiceField
                        value = getattr(self.form.cleaned_data[field], 'pk',
                                        self.form.cleaned_data[field])
                        if value == choice[0]:
                            new_choice.is_selected = True
                            selected = True
                        else:
                            new_choice.is_selected = False
                    else:
                        new_choice.is_selected = False
                    new_filter.choices.append(new_choice)

                if not self.form.fields[field].required and \
                        not [c for c in new_filter.choices if c.value == '']:
                    all_choice = Bunch(value='', label=_('All'),
                                       is_selected=(not selected))
                    new_filter.choices.insert(0, all_choice)
                filters.append(new_filter)

        return filters
<|endoftext|>


<commit_before>#! /usr/bin/python
from __future__ import print_function, division
import datetime
import matplotlib.pyplot as plt
import numpy as np
import sys

class MeterSignal(object):
    def __init__(self, filename):
        """Loads REDD-formatted data. Returns pandas Timeseries."""
        print("Loading", filename, "...", end="")
        self.data = np.loadtxt(open(filename, 'rU'), dtype=[('timestamp', np.uint32), ('watts', float)])
        self.f_diff = self.data['watts'][1:] - self.data['watts'][:-1]
        self.delta_t = self.data['timestamp'][1:] - self.data['timestamp'][:-1]
        print("done.")

    def plot(self, axes):
        x = [datetime.datetime.fromtimestamp(t) for t in self.data['timestamp']]
        axes.plot(x, self.data['watts'])

    def locate_signal(self, watts_up):
        if watts_up.duration >= self.duration:
            raise ValueError("WattsUp duration {} >= MeterSignal duration {}"
                             .format(watts_up.duration, self.duration))

        best = {'mse': np.finfo(float).max, # set Mean Sq Error to max float val
                'start_index': 0}

        last_i = self.index_from_timestamp(self.data['timestamp'][-1] - watts_up.duration)

        for start_i in range(0, last_i):
            i = start_i
            end_time = self.data['timestamp'][start_i] + watts_up.duration
            se_acc = 0 # squared error accumulator
            while self.data['timestamp'][i+1] < end_time:
                haystack_f_diff  = self.f_diff[i]
                haystack_delta_t = self.delta_t[i]

                target_start_i = self.data['timestamp'][i] - self.data['timestamp'][start_i]
                target_end_i = target_start_i + haystack_delta_t

                try:
                    target_f_diff = watts_up.data[target_end_i] - watts_up.data[target_start_i]
                except IndexError:
                    print("IndexError.", target_start_i, target_end_i, i)
                    break

                se_acc += (target_f_diff - haystack_f_diff)**2
                i += 1

            mse = se_acc / (i - start_i) # mean squared error
            if mse < best['mse']:
                best['mse'] = mse
                best['start_index'] = start_i
                print(best)

            print("{:.2%}".format(start_i / self.data.size), end="\r")
            sys.stdout.flush()

        return best

    @property
    def duration(self):
        return self.data['timestamp'][-1] - self.data['timestamp'][0]

    def index_from_timestamp(self, timestamp):
        """Return the index corresponding to timestamp.

        If timestamp is outside bounds then ValueError is raised.
        If a perfect match is then the corresponding index is returned.
        Otherwise the nearest match less than timestamp is returned.
        """

        if (timestamp > self.data['timestamp'][-1] or
            timestamp < self.data['timestamp'][0]):
            raise ValueError("timestamp {} is out of bounds".format(timestamp))

        # guess the index
        i = int(round(((timestamp - self.data['timestamp'][0]) /
                        self.duration) * (self.data.size-1)))

        # search forwards if necessary
        while self.data['timestamp'][i] < timestamp:
            i += 1

        # search backwards if necessary
        while self.data['timestamp'][i] > timestamp:
            i -= 1

        return i

class WattsUp(object):
    def __init__(self, filename, period=1):
        """Load WattsUp data"""
        print("Loading", filename)
        self.period = period # seconds
        self.data = np.loadtxt(open(filename, 'rU'), delimiter="\t", dtype=float, skiprows=3, usecols=[1])

    def plot(self, axes, start_timestamp=0):
        x = [datetime.datetime.fromtimestamp(t+start_timestamp) for t in range(0, self.data.size, self.period)]
        axes.plot(x, self.data)

    @property
    def duration(self):
        """Return duration in seconds."""
        return self.data.size * self.period

if __name__ == "__main__":
    haystack = MeterSignal("/home/jack/Dropbox/Data/data0/channel_01.dat")
    target = WattsUp("/home/jack/Dropbox/Data/BellendenWMNov2012.TXT")

    best = haystack.locate_signal(target)
    print(best)

    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    haystack.plot(ax)
    target.plot(ax, haystack.data['timestamp'][1000])
    plt.show()
    <commit_msg>Steady state code implemented. About to change locate_signal to use top 5 steady state transitions<commit_after>#! /usr/bin/python
from __future__ import print_function, division
import datetime
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
import sys

class MeterSignal(object):
    def __init__(self, filename):
        """Loads REDD-formatted data. Returns pandas Timeseries."""
        print("Loading", filename, "...", end="")
        self.data = np.loadtxt(open(filename, 'rU'), dtype=[('timestamp', np.uint32), ('watts', float)])
        self.f_diff = self.data['watts'][1:] - self.data['watts'][:-1]
        self.delta_t = self.data['timestamp'][1:] - self.data['timestamp'][:-1]
        print("done.")

    def plot(self, axes):
        x = [datetime.datetime.fromtimestamp(t) for t in self.data['timestamp']]
        axes.plot(x, self.data['watts'])

    def locate_signal(self, watts_up):
        if watts_up.duration >= self.duration:
            raise ValueError("WattsUp duration {} >= MeterSignal duration {}"
                             .format(watts_up.duration, self.duration))

        best = {'mse': np.finfo(float).max, # set Mean Sq Error to max float val
                'start_index': 0}

        last_i = self.index_from_timestamp(self.data['timestamp'][-1] - watts_up.duration)

        for start_i in range(0, last_i):
            i = start_i
            end_time = self.data['timestamp'][start_i] + watts_up.duration
            end_i = self.index_from_timestamp(end_time)

            # get the indices of the 10 largest forward diffs in this window
            top_f_diffs = np.argsort(self.f_diff[start_i:end_i])[-10:]

            se_acc = 0 # squared error accumulator
            while self.data['timestamp'][i+1] < end_time:
                haystack_f_diff  = self.f_diff[i]
                haystack_delta_t = self.delta_t[i]

                target_start_i = self.data['timestamp'][i] - self.data['timestamp'][start_i]
                target_end_i = target_start_i + haystack_delta_t

                target_f_diff = watts_up.data[target_end_i] - watts_up.data[target_start_i]

                se_acc += (target_f_diff - haystack_f_diff)**2
                i += 1

            mse = se_acc / (i - start_i) # mean squared error
            if mse < best['mse']:
                best['mse'] = mse
                best['start_index'] = start_i
                print(best)

            print("{:.2%}".format(start_i / self.data.size), end="\r")
            sys.stdout.flush()

        return best

    @property
    def duration(self):
        return self.data['timestamp'][-1] - self.data['timestamp'][0]

    def index_from_timestamp(self, timestamp):
        """Return the index corresponding to timestamp.

        If timestamp is outside bounds then ValueError is raised.
        If a perfect match is then the corresponding index is returned.
        Otherwise the nearest match less than timestamp is returned.
        """

        if (timestamp > self.data['timestamp'][-1] or
            timestamp < self.data['timestamp'][0]):
            raise ValueError("timestamp {} is out of bounds".format(timestamp))

        # guess the index
        i = int(round(((timestamp - self.data['timestamp'][0]) /
                        self.duration) * (self.data.size-1)))

        # search forwards if necessary
        while self.data['timestamp'][i] < timestamp:
            i += 1

        # search backwards if necessary
        while self.data['timestamp'][i] > timestamp:
            i -= 1

        return i

class WattsUp(object):
    def __init__(self, filename, period=1):
        """Load WattsUp data"""
        print("Loading", filename)
        self.period = period # seconds
        self.data = np.loadtxt(open(filename, 'rU'), delimiter="\t", dtype=float, skiprows=3, usecols=[1])
        self._trim()
        self.steady_states = self._calc_steady_states()
        self.steady_state_transitions = self._calc_good_steady_state_transitions()

    def plot(self, axes, start_timestamp=0):
        x = [datetime.datetime.fromtimestamp(t+start_timestamp) for t in range(0, self.data.size, self.period)]
        axes.plot(x, self.data)

    def plot_steady_states(self, axes):
        for start, end, value in self.steady_states:
            line = matplotlib.lines.Line2D([start, end], [value, value], color='r')
            axes.add_line(line)

    @property
    def duration(self):
        """Return duration in seconds."""
        return self.data.size * self.period

    def _trim(self):
        """Top and tail data to remove values <= 2 watts."""
        high = self.data > 2

        for trim_front in range(0, self.data.size):
            if high[trim_front]:
                break

        for trim_back in range(self.data.size-1, trim_front, -1):
            if high[trim_back]:
                break

        self.data = self.data[trim_front:trim_back]

    def _calc_steady_states(self):
        """Return a list of 3-element tuples: (start index, end index, value)"""
        TOL = 10 # tollerance in Watts
        MIN_LEN = 12 # min length of a steady state
        av = self.data[0]
        acc = self.data[0] # accumulator (for average)
        start_i = 0
        states = []
        for i in range(1, self.data.size):
            if av-TOL < self.data[i] < av+TOL:
                acc += self.data[i]
                av = acc / (1 + i - start_i)
            else:
                if (i-start_i) > MIN_LEN:
                    states.append((start_i, i, av))
                acc = self.data[i]
                av = self.data[i]
                start_i = i

        return states

    def _calc_good_steady_state_transitions(self):
        """Good state transitions are those which are large in magnitude
        and where the two states are very close"""
        ss_transitions = [] # list of tuples for transition: start, length, value, score
        last_ss_end = 0
        last_ss_value = 0
        for ss_start, ss_end, ss_value in self.steady_states:
            t_start = last_ss_end
            t_length = ss_start - last_ss_end
            if t_length == 0:
                t_length = 1
            t_value = ss_value - last_ss_value
            t_score = abs(t_value) if t_length < 18 else abs(t_value)/t_length

            ss_transitions.append((t_start, t_length, t_value, t_score))
            last_ss_end = ss_end
            last_ss_value = ss_value

        return sorted(ss_transitions, key=lambda score: score[3], reverse=True)

if __name__ == "__main__":
    haystack = MeterSignal("/home/jack/Dropbox/Data/data0/channel_01.dat")
    target = WattsUp("/home/jack/Dropbox/Data/BellendenWMNov2012.TXT")

    print(target.steady_states)
    print(target.steady_state_transitions)

    #best = haystack.locate_signal(target)
    #print(best)

    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)

    ax.plot(target.data)
    target.plot_steady_states(ax)

    #haystack.plot(ax)
    #target.plot(ax, haystack.data['timestamp'][25387])
    plt.show()
    <|endoftext|>


<commit_before>from django.core.mail import EmailMultiAlternatives
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.html import strip_tags
from celery.decorators import task

from djangobb_forum import settings as forum_settings
from djangobb_forum.util import absolute_url
from djangobb_forum.models import Post

from scratchr2.notifications.models import SocialAction

if "mailer" in settings.INSTALLED_APPS:
    from mailer import send_mail
else:
    from django.core.mail import send_mail
    def send_mail(subject, text, from_email, rec_list, html=None):
        """
        Shortcut for sending email.
        """

        msg = EmailMultiAlternatives(subject, text, from_email, rec_list)
        if html:
            msg.attach_alternative(html, "text/html")
        if forum_settings.EMAIL_DEBUG:
            print '---begin---'
            print 'To:', rec_list
            print 'Subject:', subject
            print 'Body:', text
            print '---end---'
        else:
            msg.send(fail_silently=True)

# TODO: move to txt template
TOPIC_SUBSCRIPTION_TEXT_TEMPLATE = (u"""New reply from %(username)s to topic that you have subscribed on.
---
%(message)s
---
See topic: %(post_url)s
Unsubscribe %(unsubscribe_url)s""")

def notify_topic_subscribers(post):
    # notify users asynchronously
    scratch_notify_topic_subscribers.delay(post.id)
    # debug version:
    # scratch_notify_topic_subscribers(post.id)

def email_topic_subscribers(post):
    """
    This is the built-in djangobb_forum method of notifying for subscriptions.
    """
    topic = post.topic
    post_body_text = strip_tags(post.body_html)
    if post != topic.head:
        subject = u'RE: %s' % topic.name
        to_email = user.email
        text_content = TOPIC_SUBSCRIPTION_TEXT_TEMPLATE % {
                'username': post.user.username,
                'message': post_body_text,
                'post_url': absolute_url(post.get_absolute_url()),
                'unsubscribe_url': absolute_url(reverse('djangobb:forum_delete_subscription', args=[post.topic.id])),
            }
        #html_content = html_version(post)
        send_mail(subject, text_content, settings.DEFAULT_FROM_EMAIL, [to_email])

@task
def scratch_notify_topic_subscribers(post_id):
    """
    Scratch task for notifying subscribers to a topic that someone has made a
    new post.
    """
    post = Post.objects.select_related('topic').get(pk=post_id)
    topic = post.topic
    if post != topic.head:
        social_action = SocialAction(
            actor = post.user,
            object = topic,
        )
        social_action.save()
        # save() adds recipients and sends out notifications
<commit_msg>remove scratchr2<commit_after>from django.core.mail import EmailMultiAlternatives
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.html import strip_tags
from celery.decorators import task

from djangobb_forum import settings as forum_settings
from djangobb_forum.util import absolute_url
from djangobb_forum.models import Post

from notifications.models import SocialAction

if "mailer" in settings.INSTALLED_APPS:
    from mailer import send_mail
else:
    from django.core.mail import send_mail
    def send_mail(subject, text, from_email, rec_list, html=None):
        """
        Shortcut for sending email.
        """

        msg = EmailMultiAlternatives(subject, text, from_email, rec_list)
        if html:
            msg.attach_alternative(html, "text/html")
        if forum_settings.EMAIL_DEBUG:
            print '---begin---'
            print 'To:', rec_list
            print 'Subject:', subject
            print 'Body:', text
            print '---end---'
        else:
            msg.send(fail_silently=True)

# TODO: move to txt template
TOPIC_SUBSCRIPTION_TEXT_TEMPLATE = (u"""New reply from %(username)s to topic that you have subscribed on.
---
%(message)s
---
See topic: %(post_url)s
Unsubscribe %(unsubscribe_url)s""")

def notify_topic_subscribers(post):
    # notify users asynchronously
    scratch_notify_topic_subscribers.delay(post.id)
    # debug version:
    # scratch_notify_topic_subscribers(post.id)

def email_topic_subscribers(post):
    """
    This is the built-in djangobb_forum method of notifying for subscriptions.
    """
    topic = post.topic
    post_body_text = strip_tags(post.body_html)
    if post != topic.head:
        subject = u'RE: %s' % topic.name
        to_email = user.email
        text_content = TOPIC_SUBSCRIPTION_TEXT_TEMPLATE % {
                'username': post.user.username,
                'message': post_body_text,
                'post_url': absolute_url(post.get_absolute_url()),
                'unsubscribe_url': absolute_url(reverse('djangobb:forum_delete_subscription', args=[post.topic.id])),
            }
        #html_content = html_version(post)
        send_mail(subject, text_content, settings.DEFAULT_FROM_EMAIL, [to_email])

@task
def scratch_notify_topic_subscribers(post_id):
    """
    Scratch task for notifying subscribers to a topic that someone has made a
    new post.
    """
    post = Post.objects.select_related('topic').get(pk=post_id)
    topic = post.topic
    if post != topic.head:
        social_action = SocialAction(
            actor = post.user,
            object = topic,
        )
        social_action.save()
        # save() adds recipients and sends out notifications
<|endoftext|>


<commit_before>from datetime import timedelta

from django.core import serializers
from django.db.models import Q
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render
from django.views.generic import View, TemplateView, CreateView, ListView
from django.views.generic.detail import DetailView

from .models import *
from .forms import SearchForm

class Index(TemplateView):
	template_name = "home.html"

	def render_to_response(self, context, **response_kwargs):
		# recepie_list = Recipe.objects.all().order_by('like', 'date')[:10]
		# context['recepies'] = recepie_list
		return super(Index, self).render_to_response(context, **response_kwargs)

# ----- korisnici -----

class RegistrationView(TemplateView):
	template_name = "registration/registration.html"

class ProfileView(DetailView):
	template_name = "profile.html"
	model = CustomUser

class LoginView(View):
	def post(self, request, *args, **kwargs):
	# logirati korisnika
		return HttpResponseRedirect(reverse('profile'))

class LogoutView(View):
	def post(self, request, *args, **kwargs):
	# odlogirati korisnika
		return HttpResponseRedirect((reverse('index')))

# ----- recepti -----

class RecipesView(TemplateView):
	template_name = 'recipes.html'

	def get_context_data(self, **kwargs):
		context = super(RecipesView, self).get_context_data(**kwargs)
		limit = 3
		recepies = Recipe.objects.all().order_by('-date')
		if recepies.count() > limit:
			more = True
		else:
			more = False

		context['recipes'] = recepies[:limit]
		context['more'] = more
		return context

class RecipeCreateView(CreateView):
	template_name = "recipe_create.html"
	model = Recipe

class RecipeDetailView(DetailView):
	template_name = "recipe_detail.html"
	model = Recipe

class RecipeSearchView(ListView):
	template_name = "recipe_search.html"
	model = Recipe

	def get_queryset(self):
		queryset = []
		filters_categories = Q()

		categories = self.request.GET.getlist('categories', [])
		tags = self.request.GET.get('tags', '')

		self.result = False
		if categories == [] and tags == '':
			return []

		for cat in categories:
			if cat != 'none':
				filters_categories |= Q(recipe_categories__categorie__name__icontains=cat)

		filters_tags = Q()
		if tags:
			tags_list = tags.split(', ')
			for tag in tags_list:
				filters_tags |= Q(recipe_tags__tag__name__icontains=tag)

		queryset = Recipe.objects.filter(filters_tags, filters_categories).distinct().order_by('-date')

		if queryset.count() != 0:
			self.result = True

		return queryset

	def get_context_data(self, **kwargs):
		context = super(RecipeSearchView, self).get_context_data(**kwargs)
		context['form'] = SearchForm()
		context['result'] = self.result
		return context

# ----- akcije -----

class DiscountCreateView(CreateView):
	template_name = "discount_create.html"
	# model = Discount

class DiscountListView(ListView):
	template_name = "discount_list.html"
	model = Discount

	def get_queryset(self):
		limit = datetime.datetime.now() - timedelta(days=7)
		queryset = Discount.objects.filter(date__gte=limit).order_by('-date')
		return queryset

	def get_context_data(self, **kwargs):
		context = super(DiscountListView, self).get_context_data(**kwargs)
		context['discounts'] = [i for i in range(20)]
		return context

class TagsView(View):
	def get(self, *args, **kwargs):
		tags = Tag.objects.all()
		data = serializers.serialize('json', tags)
		return HttpResponse(data, mimetype='application/json')

class MoreRecipesView(TemplateView):
	template_name = "more_recipes.html"

	def get_context_data(self, **kwargs):
		current_page = int(self.request.GET.get('page', 1))

		context = super(MoreRecipesView, self).get_context_data(**kwargs)
		recepies = Recipe.objects.all().order_by('-date')

		limit = 3
		more = 3
		limit_from = limit + more * (current_page - 1)
		limit_to = limit_from + more

		if recepies.count() > limit_to:
			more = True
		else:
			more = False

		context['recipes'] = recepies[limit_from:limit_to]
		context['more'] = more
		context['page'] = current_page + 1
		return context
<commit_msg>obrisani nekorišteni vieowi<commit_after>from datetime import timedelta

from django.core import serializers
from django.db.models import Q
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render
from django.views.generic import View, TemplateView, CreateView, ListView
from django.views.generic.detail import DetailView

from .models import *
from .forms import SearchForm

class Index(TemplateView):
	template_name = "home.html"

	def render_to_response(self, context, **response_kwargs):
		# recepie_list = Recipe.objects.all().order_by('like', 'date')[:10]
		# context['recepies'] = recepie_list
		return super(Index, self).render_to_response(context, **response_kwargs)

# ----- korisnici -----

class RegistrationView(CreateView):
	template_name = "registration/registration.html"

class ProfileView(DetailView):
	template_name = "profile.html"
	model = CustomUser

# ----- recepti -----

class RecipesView(TemplateView):
	template_name = 'recipes.html'

	def get_context_data(self, **kwargs):
		context = super(RecipesView, self).get_context_data(**kwargs)
		limit = 3
		recepies = Recipe.objects.all().order_by('-date')
		if recepies.count() > limit:
			more = True
		else:
			more = False

		context['recipes'] = recepies[:limit]
		context['more'] = more
		return context

class RecipeCreateView(CreateView):
	template_name = "recipe_create.html"
	model = Recipe

class RecipeDetailView(DetailView):
	template_name = "recipe_detail.html"
	model = Recipe

class RecipeSearchView(ListView):
	template_name = "recipe_search.html"
	model = Recipe

	def get_queryset(self):
		queryset = []
		filters_categories = Q()

		categories = self.request.GET.getlist('categories', [])
		tags = self.request.GET.get('tags', '')

		self.result = False
		if categories == [] and tags == '':
			return []

		for cat in categories:
			if cat != 'none':
				filters_categories |= Q(recipe_categories__categorie__name__icontains=cat)

		filters_tags = Q()
		if tags:
			tags_list = tags.split(', ')
			for tag in tags_list:
				filters_tags |= Q(recipe_tags__tag__name__icontains=tag)

		queryset = Recipe.objects.filter(filters_tags, filters_categories).distinct().order_by('-date')

		if queryset.count() != 0:
			self.result = True

		return queryset

	def get_context_data(self, **kwargs):
		context = super(RecipeSearchView, self).get_context_data(**kwargs)
		context['form'] = SearchForm()
		context['result'] = self.result
		return context

# ----- akcije -----

class DiscountCreateView(CreateView):
	template_name = "discount_create.html"
	# model = Discount

class DiscountListView(ListView):
	template_name = "discount_list.html"
	model = Discount

	def get_queryset(self):
		limit = datetime.datetime.now() - timedelta(days=7)
		queryset = Discount.objects.filter(date__gte=limit).order_by('-date')
		return queryset

	def get_context_data(self, **kwargs):
		context = super(DiscountListView, self).get_context_data(**kwargs)
		context['discounts'] = [i for i in range(20)]
		return context

class TagsView(View):
	def get(self, *args, **kwargs):
		tags = Tag.objects.all()
		data = serializers.serialize('json', tags)
		return HttpResponse(data, mimetype='application/json')

class MoreRecipesView(TemplateView):
	template_name = "more_recipes.html"

	def get_context_data(self, **kwargs):
		current_page = int(self.request.GET.get('page', 1))

		context = super(MoreRecipesView, self).get_context_data(**kwargs)
		recepies = Recipe.objects.all().order_by('-date')

		limit = 3
		more = 3
		limit_from = limit + more * (current_page - 1)
		limit_to = limit_from + more

		if recepies.count() > limit_to:
			more = True
		else:
			more = False

		context['recipes'] = recepies[limit_from:limit_to]
		context['more'] = more
		context['page'] = current_page + 1
		return context
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
from os.path import join, dirname, abspath, expanduser
parent_dir = join(dirname(__file__), "../")
sys.path.insert(0, parent_dir)

import logging
import matplotlib as plt
import xray

from biofloat import ArgoData
from biofloat.utils import o2sat, convert_to_mll
from biofloat.calibrate import (woa_o2sat, surface_mean, monthly_mean,
                                add_columns_for_groupby, add_columns_for_woa_lookup,
                                add_column_from_woa, calculate_gain
                               )

class WOA_Calibrator(object):

    logger = logging.getLogger(__name__)
    _handler = logging.StreamHandler()
    _formatter = logging.Formatter('%(levelname)s %(asctime)s %(filename)s '
                                  '%(funcName)s():%(lineno)d %(message)s')
    _handler.setFormatter(_formatter)
    logger.addHandler(_handler)

    _log_levels = (logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG)

    def make_plot(self):
        plt.style.use('ggplot')
        plt.rcParams['figure.figsize'] = (18.0, 4.0)
        gdf[['o2sat', 'woa_o2sat']].unstack(level=0).plot()
        gdf[['gain']].unstack(level=0).plot()

    def process(self):
        self.logger.setLevel(self._log_levels[self.args.verbose])
        ad = ArgoData(verbosity=self.args.verbose,
                      cache_file=self.args.cache_file)

        if self.args.wmo:
            wmo_list = self.args.wmo
        else:
            wmo_list = ad.get_cache_file_oxy_count_df()['wmo'].tolist()

        for wmo in wmo_list:
            self.logger.info('Reading float %s from %s', wmo, self.args.cache_file)
            df = ad.get_float_dataframe([wmo],
                                        max_profiles=self.args.profiles,
                                        max_pressure=self.args.pressure)

            sdf = surface_mean(df)
            sdf = add_columns_for_groupby(sdf)
            msdf = monthly_mean(sdf)
            msdf = add_columns_for_woa_lookup(msdf)
            self.logger.info('Doing WOA lookup')
            woadf = add_column_from_woa(msdf)
            gdf = calculate_gain(woadf)

            print(gdf.groupby('wmo').gain.mean())

    def process_command_line(self):
        import argparse
        from argparse import RawTextHelpFormatter

        examples = 'Examples:' + '\n'
        examples += '---------' + '\n'
        examples += sys.argv[0] + " --cache_file /data/biofloat/biofloat_fixed_cache_age365.hdf\n"
        examples += "\n\n"

        parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter,
                    description='Script to compare surface oxygen saturation values\n'
                                'with data from the World Ocean Atlas and calculate\n'
                                'a gain factor that can be used to calibrate Bio-Argo\n'
                                'float measurements.',
                    epilog=examples)

        parser.add_argument('--cache_file', action='store', help='full path to cache file',
                                            required=True)
        parser.add_argument('--wmo', action='store', nargs='*', default=[],
                                     help='One or more WMO numbers to read from cache file')
        parser.add_argument('--profiles', action='store',
                                     help='Maximum number of profiles to read in')
        parser.add_argument('--pressure', action='store',
                                     help='Maximum pressure to read in')
        parser.add_argument('-v', '--verbose', nargs='?', choices=[0,1,2,3], type=int,
                            help='0: ERROR, 1: WARN, 2: INFO, 3:DEBUG', default=0, const=2)

        self.args = parser.parse_args()

if __name__ == '__main__':

    woac = WOA_Calibrator()
    woac.process_command_line()
    woac.process()

<commit_msg>Add logger output and --results_file option (not yet implemented)<commit_after>#!/usr/bin/env python

import sys
from os.path import join, dirname, abspath, expanduser
parent_dir = join(dirname(__file__), "../")
sys.path.insert(0, parent_dir)

import logging
import matplotlib as plt
import xray

from biofloat import ArgoData
from biofloat.utils import o2sat, convert_to_mll
from biofloat.calibrate import (woa_o2sat, surface_mean, monthly_mean,
                                add_columns_for_groupby, add_columns_for_woa_lookup,
                                add_column_from_woa, calculate_gain
                               )

class WOA_Calibrator(object):

    logger = logging.getLogger(__name__)
    _handler = logging.StreamHandler()
    _formatter = logging.Formatter('%(levelname)s %(asctime)s %(filename)s '
                                  '%(funcName)s():%(lineno)d %(message)s')
    _handler.setFormatter(_formatter)
    logger.addHandler(_handler)

    _log_levels = (logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG)

    def make_plot(self):
        plt.style.use('ggplot')
        plt.rcParams['figure.figsize'] = (18.0, 4.0)
        gdf[['o2sat', 'woa_o2sat']].unstack(level=0).plot()
        gdf[['gain']].unstack(level=0).plot()

    def process(self):
        self.logger.setLevel(self._log_levels[self.args.verbose])
        ad = ArgoData(verbosity=self.args.verbose,
                      cache_file=self.args.cache_file)

        if self.args.wmo:
            wmo_list = self.args.wmo
        else:
            wmo_list = ad.get_cache_file_oxy_count_df()['wmo'].tolist()

        for wmo in wmo_list:
            self.logger.info('Reading float %s from %s', wmo, self.args.cache_file)
            df = ad.get_float_dataframe([wmo],
                                        max_profiles=self.args.profiles,
                                        max_pressure=self.args.pressure)

            sdf = surface_mean(df)
            sdf = add_columns_for_groupby(sdf)
            msdf = monthly_mean(sdf)
            msdf = add_columns_for_woa_lookup(msdf)
            self.logger.info('Doing WOA lookup for %s points', len(msdf))
            woadf = add_column_from_woa(msdf, verbose=self.args.verbose)
            gdf = calculate_gain(woadf)

            self.logger.info('Gain for %s = %s', wmo, gdf.groupby('wmo').gain.mean().values[0])
            # TODO: Save to HDf file

    def process_command_line(self):
        import argparse
        from argparse import RawTextHelpFormatter

        examples = 'Examples:' + '\n'
        examples += '---------' + '\n'
        examples += sys.argv[0] + " --cache_file /data/biofloat/biofloat_fixed_cache_age365.hdf\n"
        examples += "\n\n"

        parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter,
                    description='Script to compare surface oxygen saturation values\n'
                                'with data from the World Ocean Atlas and calculate\n'
                                'a gain factor that can be used to calibrate Bio-Argo\n'
                                'float measurements.',
                    epilog=examples)

        parser.add_argument('--cache_file', action='store', help='full path to cache file',
                                            required=True)
        parser.add_argument('--wmo', action='store', nargs='*', default=[],
                                     help='One or more WMO numbers to read from cache file')
        parser.add_argument('--profiles', action='store',
                                     help='Maximum number of profiles to read in')
        parser.add_argument('--pressure', action='store',
                                     help='Maximum pressure to read in')
        parser.add_argument('--results_file', action='store',
                                     help='File name to save df of the results')
        parser.add_argument('-v', '--verbose', nargs='?', choices=[0,1,2,3], type=int,
                            help='0: ERROR, 1: WARN, 2: INFO, 3:DEBUG', default=0, const=2)

        self.args = parser.parse_args()

if __name__ == '__main__':

    woac = WOA_Calibrator()
    woac.process_command_line()
    woac.process()

<|endoftext|>


<commit_before>from djangorestframework.mediatypes import MediaType
#from djangorestframework.requestparsing import parse, load_parser
from StringIO import StringIO

class RequestMixin(object):
    """Delegate class that supplements an HttpRequest object with additional behaviour."""

    USE_FORM_OVERLOADING = True
    METHOD_PARAM = "_method"
    CONTENTTYPE_PARAM = "_content_type"
    CONTENT_PARAM = "_content"

    def _get_method(self):
        """
        Returns the HTTP method for the current view.
        """
        if not hasattr(self, '_method'):
            self._method = self.request.method
        return self._method

    def _set_method(self, method):
        """
        Set the method for the current view.
        """
        self._method = method

    def _get_content_type(self):
        """
        Returns a MediaType object, representing the request's content type header.
        """
        if not hasattr(self, '_content_type'):
            content_type = self.request.META.get('HTTP_CONTENT_TYPE', self.request.META.get('CONTENT_TYPE', ''))
            self._content_type = MediaType(content_type)
        return self._content_type

    def _set_content_type(self, content_type):
        """
        Set the content type.  Should be a MediaType object.
        """
        self._content_type = content_type

    def _get_accept(self):
        """
        Returns a list of MediaType objects, representing the request's accept header.
        """
        if not hasattr(self, '_accept'):
            accept = self.request.META.get('HTTP_ACCEPT', '*/*')
            self._accept = [MediaType(elem) for elem in accept.split(',')]
        return self._accept

    def _set_accept(self):
        """
        Set the acceptable media types.  Should be a list of MediaType objects.
        """
        self._accept = accept

    def _get_stream(self):
        """
        Returns an object that may be used to stream the request content.
        """
        if not hasattr(self, '_stream'):
            if hasattr(self.request, 'read'):
                self._stream = self.request
            else:
                self._stream = StringIO(self.request.raw_post_data)
        return self._stream

    def _set_stream(self, stream):
        """
        Set the stream representing the request body.
        """
        self._stream = stream

    def _get_raw_content(self):
        """
        Returns the parsed content of the request
        """
        if not hasattr(self, '_raw_content'):
            self._raw_content = self.parse(self.stream, self.content_type)
        return self._raw_content

    def _get_content(self):
        """
        Returns the parsed and validated content of the request
        """
        if not hasattr(self, '_content'):
            self._content = self.validate(self.RAW_CONTENT)

        return self._content

    def perform_form_overloading(self):
        """
        Check the request to see if it is using form POST '_method'/'_content'/'_content_type' overrides.
        If it is then alter self.method, self.content_type, self.CONTENT to reflect that rather than simply
        delegating them to the original request.
        """
        if not self.USE_FORM_OVERLOADING or self.method != 'POST' or not self.content_type.is_form():
            return

        content = self.RAW_CONTENT
        if self.METHOD_PARAM in content:
            self.method = content[self.METHOD_PARAM].upper()
            del self._raw_content[self.METHOD_PARAM]

        if self.CONTENT_PARAM in content and self.CONTENTTYPE_PARAM in content:
            self._content_type = MediaType(content[self.CONTENTTYPE_PARAM])
            self._stream = StringIO(content[self.CONTENT_PARAM])
            del(self._raw_content)

    method = property(_get_method, _set_method)
    content_type = property(_get_content_type, _set_content_type)
    accept = property(_get_accept, _set_accept)
    stream = property(_get_stream, _set_stream)
    RAW_CONTENT = property(_get_raw_content)
    CONTENT = property(_get_content)

<commit_msg>Temporarily roll back the streaming stuff<commit_after>from djangorestframework.mediatypes import MediaType
#from djangorestframework.requestparsing import parse, load_parser
from django.http.multipartparser import LimitBytes
from StringIO import StringIO

class RequestMixin(object):
    """Delegate class that supplements an HttpRequest object with additional behaviour."""

    USE_FORM_OVERLOADING = True
    METHOD_PARAM = "_method"
    CONTENTTYPE_PARAM = "_content_type"
    CONTENT_PARAM = "_content"

    def _get_method(self):
        """
        Returns the HTTP method for the current view.
        """
        if not hasattr(self, '_method'):
            self._method = self.request.method
        return self._method

    def _set_method(self, method):
        """
        Set the method for the current view.
        """
        self._method = method

    def _get_content_type(self):
        """
        Returns a MediaType object, representing the request's content type header.
        """
        if not hasattr(self, '_content_type'):
            content_type = self.request.META.get('HTTP_CONTENT_TYPE', self.request.META.get('CONTENT_TYPE', ''))
            self._content_type = MediaType(content_type)
        return self._content_type

    def _set_content_type(self, content_type):
        """
        Set the content type.  Should be a MediaType object.
        """
        self._content_type = content_type

    def _get_accept(self):
        """
        Returns a list of MediaType objects, representing the request's accept header.
        """
        if not hasattr(self, '_accept'):
            accept = self.request.META.get('HTTP_ACCEPT', '*/*')
            self._accept = [MediaType(elem) for elem in accept.split(',')]
        return self._accept

    def _set_accept(self):
        """
        Set the acceptable media types.  Should be a list of MediaType objects.
        """
        self._accept = accept

    def _get_stream(self):
        """
        Returns an object that may be used to stream the request content.
        """
        if not hasattr(self, '_stream'):
            request = self.request
            # We ought to be able to return a stream rather than reading the stream.
            # Not quite working just yet...
            #if hasattr(request, 'read'):
            #    try:
            #        content_length = int(request.META.get('CONTENT_LENGTH',0))
            #    except (ValueError, TypeError):
            #        content_length = 0
            #    self._stream = LimitBytes(request, content_length)
            #else:
            self._stream = StringIO(request.raw_post_data)
        return self._stream

    def _set_stream(self, stream):
        """
        Set the stream representing the request body.
        """
        self._stream = stream

    def _get_raw_content(self):
        """
        Returns the parsed content of the request
        """
        if not hasattr(self, '_raw_content'):
            self._raw_content = self.parse(self.stream, self.content_type)
        return self._raw_content

    def _get_content(self):
        """
        Returns the parsed and validated content of the request
        """
        if not hasattr(self, '_content'):
            self._content = self.validate(self.RAW_CONTENT)

        return self._content

    def perform_form_overloading(self):
        """
        Check the request to see if it is using form POST '_method'/'_content'/'_content_type' overrides.
        If it is then alter self.method, self.content_type, self.CONTENT to reflect that rather than simply
        delegating them to the original request.
        """
        if not self.USE_FORM_OVERLOADING or self.method != 'POST' or not self.content_type.is_form():
            return

        content = self.RAW_CONTENT
        if self.METHOD_PARAM in content:
            self.method = content[self.METHOD_PARAM].upper()
            del self._raw_content[self.METHOD_PARAM]

        if self.CONTENT_PARAM in content and self.CONTENTTYPE_PARAM in content:
            self._content_type = MediaType(content[self.CONTENTTYPE_PARAM])
            self._stream = StringIO(content[self.CONTENT_PARAM])
            del(self._raw_content)

    method = property(_get_method, _set_method)
    content_type = property(_get_content_type, _set_content_type)
    accept = property(_get_accept, _set_accept)
    stream = property(_get_stream, _set_stream)
    RAW_CONTENT = property(_get_raw_content)
    CONTENT = property(_get_content)

<|endoftext|>


<commit_before>"""

Copyright 2015, Institute e-Austria, Timisoara, Romania
    http://www.ieat.ro/
Developers:
 * Gabriel Iuhasz, iuhasz.gabriel@info.uvt.ro

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at:
    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from flask import Flask
from flask import jsonify
from flask import send_file
from flask import request
from flask.ext.restplus import Api, Resource, fields
from flask import abort
import os
import jinja2
import sys
import subprocess
import platform
import requests
import json

from pyLogstash import *
from jsonvalidation import *
from jsonschema import *

#directory location

tmpDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')
pidDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'pid')
logDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'log')
cfgDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config')
credDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'credentials')
lockDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lock')
logstashDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logstash')

app = Flask("dmon-logstash")
app.config['RESTPLUS_VALIDATE'] = True
api = Api(app, version='0.0.1', title='DICE Monitoring Logstash API',
          description="RESTful API for the DICE Monitoring Platform  Logstash agent (dmon-logstash)",
          )

# changes the descriptor on the Swagger WUI and appends to api /dmon and then /v1
agent = api.namespace('agent', description='dmon logstash operations')

lsConfig = api.model('LS Configuration options', {
    'UDPPort': fields.String(required=True, default='25826', description='Port of UDP plugin from Logstash Server'),
    'ESCluster': fields.String(required=True, default='esCore', description='Name of ES cluster'),
    'EShostIP': fields.String(required=True, default='localhost', description='ES endpoint IP'),
    'EShostPort': fields.String(required=True, default='9200', description='ES endpoint Port'),
    'LSWorkers': fields.String(required=True, default='2', description='Number of workers'),
    'LSHeap': fields.String(required=True, default='512m', description='Size of JVM Heap'),
    'StormRestIP': fields.String(required=False, default='none', description='Storm REST API endpoint IP'),
    'StormRestPort': fields.String(required=False, default='none', description='Storm REST API endpoint Port'),
    'StormTopologyID': fields.String(required=False, default='none', description='Storm Topology')
})

lsagent = pyLogstashInstance()
valid = LSValidation()

@agent.route('/v1/host')
class NodeInfo(Resource):
    def get(self):
        mType = platform.uname()
        response = jsonify({'System': mType[0],
                            'Node': mType[1],
                            'Release': mType[2],
                            'Version': mType[3],
                            'Machine': mType[4],
                            'Processor': mType[5]})
        response.status_code = 200
        return response

@agent.route('/v1/cert')
class LSCertificates(Resource):
    def get(self):
        dirContent = os.listdir(credDir)
        if dirContent == []:
            response = jsonify({'Status': 'Env Error',
                                'Message': 'Credential folder empty'})
            response.status_code = 404
            return response
        pubFile = []
        privateFile = []
        for f in dirContent:
           if os.path.splitext(f)[1] == 'crt':
               pubFile.append(f)
           elif os.path.splitext(f)[1] == 'key':
               privateFile.append(f)

        response = jsonify({'certificates': pubFile,
                            'keys': privateFile})
        response.status_code = 200
        return response

    def post(self):
        return "Send new certificate!"

@agent.route('/v1/logstash')
class LSStatus(Resource):
    def get(self):

        if not os.path.isfile(os.path.join(pidDir, 'logstash.pid')):
            response = jsonify({'Status': 'Env Error',
                                'Message': 'PID file not found!'})
            response.status_code = 404
            return response
        status = lsagent.check()

        if not status:
            response = jsonify({'Status': 'Stopped',
                                'Message': 'No LS instance found'})
            response.status_code = 200
            return response
        else:
            response = jsonify({'Status': 'Running',
                                'Message': 'PID ' + status})
            response.status_code = 200
            return response

@agent.route('/v1/logstash/config')
class LSController(Resource):
    def get(self):
        conf = os.path.isfile(os.path.join(cfgDir, 'logstash.conf'))
        if not conf:
            response = jsonify({'Status': 'Env Error',
                                'Message': 'No config found!'})
            response.status_code = 404
            return response

        confFile = open(conf, 'r')

        return send_file(confFile, mimetype='text/plain', as_attachment=True)

    @api.expect(lsConfig)
    def post(self):
        sslCert = os.path.join(credDir, 'logstash.crt')
        sslKey = os.path.join(credDir, 'logstash.key')

        if not os.path.isfile(sslCert) or not os.path.isfile(sslKey):
            response = jsonify({'Status': 'Credential Error',
                                'Message': 'Missing keys'})
            response.status_code = 404
            return response
        if not request.json:
            abort(400)

        try:
            valid.validate(request.json)
        except ValidationError:
            response = jsonify({'Status': 'json error',
                                'Message': 'Malformed json'})
            response.status_code = 404
            return response

        if not request.json['StormRestIP']:
            StormRestIP = 'none'
        else:
            StormRestIP = request.json['StormRestIP']

        if not request.json['StormRestPort']:
            StormRestPort = 'none'
        else:
            StormRestPort = request.json['StormRestPort']

        if not request.json['StormTopologyID']:
            if not request.json['StormRestIP']:
                StormTopologyID = 'none'
            else:
                if request.json['StormTopologyID'] == 'none':
                    StormTopologyID = 'none'
                else:
                    # Get first topology from storm and use this for monitoring
                    stormTopologyURL = 'http://'+StormRestIP+':'+StormRestPort+'/api/v1/topology/'
                    listTopologies = requests.get(stormTopologyURL + 'summary')
                    plainText = listTopologies.text
                    jsonPayload = json.loads(plainText)
                    topology = jsonPayload['topologies'][0]['encodedId'] #TODO: get list of topologies instead of just the first one
                    #getTopology = request.get(stormTopologyURL + topology)
                    StormTopologyID = topology

        confdict = {"sslcert": sslCert, "sslkey": sslKey, "udpPort": request.json['UDPPort'],
                    "ESCluster": request.json['ESCluster'], "EShostIP": request.json['EShostIP'], "EShostPort": request.json['EShostPort'],
                    "StormRestIP": StormRestIP, "StormRestPort": StormRestPort, "StormTopologyID": StormTopologyID}
        lsagent.generateConfig(confdict)
        pid = lsagent.check()
        if not pid:
            lsagent.start(heap=request.json['LSHeap'], worker=request.json['LSWorker'])
        else:
            subprocess.call(['kill', '-9', pid])
            lsagent.start(heap=request.json['LSHeap'], worker=request.json['LSWorker'])

        response = jsonify({'Status': 'Done',
                            'Message': 'LS config loaded'})
        response.status_code = 200
        return response

@agent.route('/v1/logstash/start')
class LSControllerStart(Resource):
    def post(self):
        return "Start/Restart LS instance!"

@agent.route('/v1/logstash/stop')
class LSControllerStop(Resource):
    def post(self):
        pid = lsagent.check()
        if not pid:
            response = jsonify({'Status': 'Not Found',
                                'Message': 'LS instance not found'})
            response.status_code = 404
            return response
        else:
            subprocess.call(['kill', '-9', pid])
            response = jsonify({'Status': 'Done',
                                'Message': 'LS Instance stopped'})
            response.status_code = 200
            return response

@agent.route('/v1/logstash/deploy')
class LSControllerDeploy(Resource):
    def post(self):
        lsagent.deploy()
        response = jsonify({'Status': 'Done',
                            'Message': 'Logstash installed!'})
        response.status_code = 201
        return response

@agent.route('/v1/logstash/log')
class LSControllerLog(Resource):
    def get(self):
        lslog = os.path.join(logDir, 'logstash.log')

        if not os.path.isfile(lslog):
            response = jsonify({'Status': 'Env Error',
                                'Message': 'No log file found'})
            response.status_code = 404
            return response

        sizeMB = os.path.getsize(lslog) >> 20
        if sizeMB > 10:
            response = jsonify({'Status': 'Size Warning',
                                'Message': 'Log file to big!'})
            response.status_code = 413
            return response

        logCont = open(lslog, 'r')
        return send_file(logCont, mimetype='text/plain', as_attachment=True)

if __name__ == '__main__':
    app.run(host='0.0.0.0', debug=True)
<commit_msg>fix json parsing<commit_after>"""

Copyright 2015, Institute e-Austria, Timisoara, Romania
    http://www.ieat.ro/
Developers:
 * Gabriel Iuhasz, iuhasz.gabriel@info.uvt.ro

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at:
    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from flask import Flask
from flask import jsonify
from flask import send_file
from flask import request
from flask.ext.restplus import Api, Resource, fields
from flask import abort
import os
import jinja2
import sys
import subprocess
import platform
import requests
import json

from pyLogstash import *
from jsonvalidation import *
from jsonschema import *

#directory location

tmpDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')
pidDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'pid')
logDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'log')
cfgDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config')
credDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'credentials')
lockDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lock')
logstashDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logstash')

app = Flask("dmon-logstash")
app.config['RESTPLUS_VALIDATE'] = True
api = Api(app, version='0.0.1', title='DICE Monitoring Logstash API',
          description="RESTful API for the DICE Monitoring Platform  Logstash agent (dmon-logstash)",
          )

# changes the descriptor on the Swagger WUI and appends to api /dmon and then /v1
agent = api.namespace('agent', description='dmon logstash operations')

lsConfig = api.model('LS Configuration options', {
    'UDPPort': fields.String(required=True, default='25826', description='Port of UDP plugin from Logstash Server'),
    'ESCluster': fields.String(required=True, default='esCore', description='Name of ES cluster'),
    'EShostIP': fields.String(required=True, default='localhost', description='ES endpoint IP'),
    'EShostPort': fields.String(required=True, default='9200', description='ES endpoint Port'),
    'LSWorkers': fields.String(required=True, default='2', description='Number of workers'),
    'LSHeap': fields.String(required=True, default='512m', description='Size of JVM Heap'),
    'StormRestIP': fields.String(required=False, default='none', description='Storm REST API endpoint IP'),
    'StormRestPort': fields.String(required=False, default='none', description='Storm REST API endpoint Port'),
    'StormTopologyID': fields.String(required=False, default='none', description='Storm Topology')
})

lsagent = pyLogstashInstance()
valid = LSValidation()

@agent.route('/v1/host')
class NodeInfo(Resource):
    def get(self):
        mType = platform.uname()
        response = jsonify({'System': mType[0],
                            'Node': mType[1],
                            'Release': mType[2],
                            'Version': mType[3],
                            'Machine': mType[4],
                            'Processor': mType[5]})
        response.status_code = 200
        return response

@agent.route('/v1/cert')
class LSCertificates(Resource):
    def get(self):
        dirContent = os.listdir(credDir)
        if dirContent == []:
            response = jsonify({'Status': 'Env Error',
                                'Message': 'Credential folder empty'})
            response.status_code = 404
            return response
        pubFile = []
        privateFile = []
        for f in dirContent:
           if os.path.splitext(f)[1] == 'crt':
               pubFile.append(f)
           elif os.path.splitext(f)[1] == 'key':
               privateFile.append(f)

        response = jsonify({'certificates': pubFile,
                            'keys': privateFile})
        response.status_code = 200
        return response

    def post(self):
        return "Send new certificate!"

@agent.route('/v1/logstash')
class LSStatus(Resource):
    def get(self):

        if not os.path.isfile(os.path.join(pidDir, 'logstash.pid')):
            response = jsonify({'Status': 'Env Error',
                                'Message': 'PID file not found!'})
            response.status_code = 404
            return response
        status = lsagent.check()

        if not status:
            response = jsonify({'Status': 'Stopped',
                                'Message': 'No LS instance found'})
            response.status_code = 200
            return response
        else:
            response = jsonify({'Status': 'Running',
                                'Message': 'PID ' + status})
            response.status_code = 200
            return response

@agent.route('/v1/logstash/config')
class LSController(Resource):
    def get(self):
        conf = os.path.isfile(os.path.join(cfgDir, 'logstash.conf'))
        if not conf:
            response = jsonify({'Status': 'Env Error',
                                'Message': 'No config found!'})
            response.status_code = 404
            return response

        confFile = open(conf, 'r')

        return send_file(confFile, mimetype='text/plain', as_attachment=True)

    @api.expect(lsConfig)
    def post(self):
        sslCert = os.path.join(credDir, 'logstash.crt')
        sslKey = os.path.join(credDir, 'logstash.key')

        if not os.path.isfile(sslCert) or not os.path.isfile(sslKey):
            response = jsonify({'Status': 'Credential Error',
                                'Message': 'Missing keys'})
            response.status_code = 404
            return response
        if not request.json:
            abort(400)

        try:
            valid.validate(request.json)
        except ValidationError:
            response = jsonify({'Status': 'json error',
                                'Message': 'Malformed json'})
            response.status_code = 404
            return response

        if not 'StormRestIP' in request.json:
            StormRestIP = 'none'
        else:
            StormRestIP = request.json['StormRestIP']

        if not 'StormRestPort' in request.json:
            StormRestPort = 'none'
        else:
            StormRestPort = request.json['StormRestPort']

        if not 'StormTopologyID' in request.json:
            if not request.json['StormRestIP']:
                StormTopologyID = 'none'
            else:
                if request.json['StormTopologyID'] == 'none':
                    StormTopologyID = 'none'
                else:
                    # Get first topology from storm and use this for monitoring
                    stormTopologyURL = 'http://'+StormRestIP+':'+StormRestPort+'/api/v1/topology/'
                    listTopologies = requests.get(stormTopologyURL + 'summary')
                    plainText = listTopologies.text
                    jsonPayload = json.loads(plainText)
                    topology = jsonPayload['topologies'][0]['encodedId'] #TODO: get list of topologies instead of just the first one
                    #getTopology = request.get(stormTopologyURL + topology)
                    StormTopologyID = topology

        confdict = {"sslcert": sslCert, "sslkey": sslKey, "udpPort": request.json['UDPPort'],
                    "ESCluster": request.json['ESCluster'], "EShostIP": request.json['EShostIP'], "EShostPort": request.json['EShostPort'],
                    "StormRestIP": StormRestIP, "StormRestPort": StormRestPort, "StormTopologyID": StormTopologyID}
        lsagent.generateConfig(confdict)
        pid = lsagent.check()
        if not pid:
            lsagent.start(heap=request.json['LSHeap'], worker=request.json['LSWorker'])
        else:
            subprocess.call(['kill', '-9', pid])
            lsagent.start(heap=request.json['LSHeap'], worker=request.json['LSWorker'])

        response = jsonify({'Status': 'Done',
                            'Message': 'LS config loaded'})
        response.status_code = 200
        return response

@agent.route('/v1/logstash/start')
class LSControllerStart(Resource):
    def post(self):
        return "Start/Restart LS instance!"

@agent.route('/v1/logstash/stop')
class LSControllerStop(Resource):
    def post(self):
        pid = lsagent.check()
        if not pid:
            response = jsonify({'Status': 'Not Found',
                                'Message': 'LS instance not found'})
            response.status_code = 404
            return response
        else:
            subprocess.call(['kill', '-9', pid])
            response = jsonify({'Status': 'Done',
                                'Message': 'LS Instance stopped'})
            response.status_code = 200
            return response

@agent.route('/v1/logstash/deploy')
class LSControllerDeploy(Resource):
    def post(self):
        lsagent.deploy()
        response = jsonify({'Status': 'Done',
                            'Message': 'Logstash installed!'})
        response.status_code = 201
        return response

@agent.route('/v1/logstash/log')
class LSControllerLog(Resource):
    def get(self):
        lslog = os.path.join(logDir, 'logstash.log')

        if not os.path.isfile(lslog):
            response = jsonify({'Status': 'Env Error',
                                'Message': 'No log file found'})
            response.status_code = 404
            return response

        sizeMB = os.path.getsize(lslog) >> 20
        if sizeMB > 10:
            response = jsonify({'Status': 'Size Warning',
                                'Message': 'Log file to big!'})
            response.status_code = 413
            return response

        logCont = open(lslog, 'r')
        return send_file(logCont, mimetype='text/plain', as_attachment=True)

if __name__ == '__main__':
    app.run(host='0.0.0.0', debug=True)
<|endoftext|>


<commit_before>import logging
import os
import pickle
from typing import Optional

import numpy as np
import pandas as pd
import torch
from anndata import read

from scvi._compat import Literal
from scvi._utils import track
from scvi.core.utils import DifferentialComputation

logger = logging.getLogger(__name__)

def _load_saved_files(
    dir_path: str,
    load_adata: bool,
    map_location: Optional[Literal["cpu", "cuda"]] = None,
):
    """Helper to load saved files."""
    setup_dict_path = os.path.join(dir_path, "attr.pkl")
    adata_path = os.path.join(dir_path, "adata.h5ad")
    varnames_path = os.path.join(dir_path, "var_names.csv")
    model_path = os.path.join(dir_path, "model_params.pt")

    if os.path.exists(adata_path) and load_adata:
        adata = read(adata_path)
    elif not os.path.exists(adata_path) and load_adata:
        raise ValueError("Save path contains no saved anndata and no adata was passed.")
    else:
        adata = None

    var_names = np.genfromtxt(varnames_path, delimiter=",", dtype=str)

    with open(setup_dict_path, "rb") as handle:
        attr_dict = pickle.load(handle)

    scvi_setup_dict = attr_dict.pop("scvi_setup_dict_")

    model_state_dict = torch.load(model_path, map_location=map_location)

    return scvi_setup_dict, attr_dict, var_names, model_state_dict, adata

def _initialize_model(cls, adata, attr_dict, use_cuda):
    """Helper to initialize a model."""
    if "init_params_" not in attr_dict.keys():
        raise ValueError(
            "No init_params_ were saved by the model. Check out the "
            "developers guide if creating custom models."
        )
    # get the parameters for the class init signiture
    init_params = attr_dict.pop("init_params_")

    # update use_cuda from the saved model
    init_params["use_cuda"] = use_cuda

    # grab all the parameters execept for kwargs (is a dict)
    non_kwargs = {k: v for k, v in init_params.items() if not isinstance(v, dict)}
    # expand out kwargs
    kwargs = {k: v for k, v in init_params.items() if isinstance(v, dict)}
    kwargs = {k: v for (i, j) in kwargs.items() for (k, v) in j.items()}
    model = cls(adata, **non_kwargs, **kwargs)

    return model

def _validate_var_names(adata, source_var_names):

    user_var_names = adata.var_names.astype(str)
    if not np.array_equal(source_var_names, user_var_names):
        logger.warning(
            "var_names for adata passed in does not match var_names of "
            "adata used to train the model. For valid results, the vars "
            "need to be the same and in the same order as the adata used to train the model."
        )

def _de_core(
    adata,
    model_fn,
    groupby,
    group1,
    group2,
    idx1,
    idx2,
    all_stats,
    all_stats_fn,
    col_names,
    mode,
    batchid1,
    batchid2,
    delta,
    batch_correction,
    fdr,
    **kwargs
):
    """Internal function for DE interface."""
    if group1 is None and idx1 is None:
        group1 = adata.obs[groupby].cat.categories.tolist()
        if len(group1) == 1:
            raise ValueError(
                "Only a single group in the data. Can't run DE on a single group."
            )

    if isinstance(group1, str):
        group1 = [group1]

    # make a temp obs key using indices
    temp_key = None
    if idx1 is not None:
        idx1 = np.asarray(idx1).ravel()
        g1_key = "one"
        obs_col = np.array(["None"] * adata.shape[0], dtype=str)
        obs_col[idx1] = g1_key
        group2 = None if idx2 is None else "two"
        if idx2 is not None:
            idx2 = np.asarray(idx2).ravel()
            obs_col[idx2] = group2
        temp_key = "_scvi_temp_de"
        adata.obs[temp_key] = obs_col
        groupby = temp_key
        group1 = [g1_key]

    df_results = []
    dc = DifferentialComputation(model_fn, adata)
    for g1 in track(
        group1,
        description="DE...",
    ):
        cell_idx1 = (adata.obs[groupby] == g1).to_numpy().ravel()
        if group2 is None:
            cell_idx2 = ~cell_idx1
        else:
            cell_idx2 = (adata.obs[groupby] == group2).to_numpy().ravel()

        all_info = dc.get_bayes_factors(
            cell_idx1,
            cell_idx2,
            mode=mode,
            delta=delta,
            batchid1=batchid1,
            batchid2=batchid2,
            use_observed_batches=not batch_correction,
            **kwargs,
        )

        if all_stats is True:
            genes_properties_dict = all_stats_fn(adata, cell_idx1, cell_idx2)
            all_info = {**all_info, **genes_properties_dict}

        res = pd.DataFrame(all_info, index=col_names)
        sort_key = "proba_de" if mode == "change" else "bayes_factor"
        res = res.sort_values(by=sort_key, ascending=False)
        if mode == "change":
            res["is_de_fdr_{}".format(fdr)] = _fdr_de_prediction(
                res["proba_de"], fdr=fdr
            )
        if idx1 is None:
            g2 = "Rest" if group2 is None else group2
            res["comparison"] = "{} vs {}".format(g1, g2)
        df_results.append(res)

    if temp_key is not None:
        del adata.obs[temp_key]

    result = pd.concat(df_results, axis=0)

    return result

def _fdr_de_prediction(posterior_probas: np.ndarray, fdr: float = 0.05):
    """Compute posterior expected FDR and tag features as DE."""
    if not posterior_probas.ndim == 1:
        raise ValueError("posterior_probas should be 1-dimensional")
    sorted_genes = np.argsort(-posterior_probas)
    sorted_pgs = posterior_probas[sorted_genes]
    cumulative_fdr = (1.0 - sorted_pgs).cumsum() / (1.0 + np.arange(len(sorted_pgs)))
    d = (cumulative_fdr <= fdr).sum()
    pred_de_genes = sorted_genes[:d]
    is_pred_de = np.zeros_like(cumulative_fdr).astype(bool)
    is_pred_de[pred_de_genes] = True
    return is_pred_de
<commit_msg>group category can be int<commit_after>import logging
import os
import pickle
from typing import Optional

import numpy as np
import pandas as pd
import torch
from anndata import read
from collections.abc import Iterable as IterableClass

from scvi._compat import Literal
from scvi._utils import track
from scvi.core.utils import DifferentialComputation

logger = logging.getLogger(__name__)

def _load_saved_files(
    dir_path: str,
    load_adata: bool,
    map_location: Optional[Literal["cpu", "cuda"]] = None,
):
    """Helper to load saved files."""
    setup_dict_path = os.path.join(dir_path, "attr.pkl")
    adata_path = os.path.join(dir_path, "adata.h5ad")
    varnames_path = os.path.join(dir_path, "var_names.csv")
    model_path = os.path.join(dir_path, "model_params.pt")

    if os.path.exists(adata_path) and load_adata:
        adata = read(adata_path)
    elif not os.path.exists(adata_path) and load_adata:
        raise ValueError("Save path contains no saved anndata and no adata was passed.")
    else:
        adata = None

    var_names = np.genfromtxt(varnames_path, delimiter=",", dtype=str)

    with open(setup_dict_path, "rb") as handle:
        attr_dict = pickle.load(handle)

    scvi_setup_dict = attr_dict.pop("scvi_setup_dict_")

    model_state_dict = torch.load(model_path, map_location=map_location)

    return scvi_setup_dict, attr_dict, var_names, model_state_dict, adata

def _initialize_model(cls, adata, attr_dict, use_cuda):
    """Helper to initialize a model."""
    if "init_params_" not in attr_dict.keys():
        raise ValueError(
            "No init_params_ were saved by the model. Check out the "
            "developers guide if creating custom models."
        )
    # get the parameters for the class init signiture
    init_params = attr_dict.pop("init_params_")

    # update use_cuda from the saved model
    init_params["use_cuda"] = use_cuda

    # grab all the parameters execept for kwargs (is a dict)
    non_kwargs = {k: v for k, v in init_params.items() if not isinstance(v, dict)}
    # expand out kwargs
    kwargs = {k: v for k, v in init_params.items() if isinstance(v, dict)}
    kwargs = {k: v for (i, j) in kwargs.items() for (k, v) in j.items()}
    model = cls(adata, **non_kwargs, **kwargs)

    return model

def _validate_var_names(adata, source_var_names):

    user_var_names = adata.var_names.astype(str)
    if not np.array_equal(source_var_names, user_var_names):
        logger.warning(
            "var_names for adata passed in does not match var_names of "
            "adata used to train the model. For valid results, the vars "
            "need to be the same and in the same order as the adata used to train the model."
        )

def _de_core(
    adata,
    model_fn,
    groupby,
    group1,
    group2,
    idx1,
    idx2,
    all_stats,
    all_stats_fn,
    col_names,
    mode,
    batchid1,
    batchid2,
    delta,
    batch_correction,
    fdr,
    **kwargs
):
    """Internal function for DE interface."""
    if group1 is None and idx1 is None:
        group1 = adata.obs[groupby].cat.categories.tolist()
        if len(group1) == 1:
            raise ValueError(
                "Only a single group in the data. Can't run DE on a single group."
            )

    # if isinstance(group1, str):
    #     group1 = [group1]
    if not isinstance(group1, IterableClass) or isinstance(group1, str):
        group1 = [group1]

    # make a temp obs key using indices
    temp_key = None
    if idx1 is not None:
        idx1 = np.asarray(idx1).ravel()
        g1_key = "one"
        obs_col = np.array(["None"] * adata.shape[0], dtype=str)
        obs_col[idx1] = g1_key
        group2 = None if idx2 is None else "two"
        if idx2 is not None:
            idx2 = np.asarray(idx2).ravel()
            obs_col[idx2] = group2
        temp_key = "_scvi_temp_de"
        adata.obs[temp_key] = obs_col
        groupby = temp_key
        group1 = [g1_key]

    df_results = []
    dc = DifferentialComputation(model_fn, adata)
    for g1 in track(
        group1,
        description="DE...",
    ):
        cell_idx1 = (adata.obs[groupby] == g1).to_numpy().ravel()
        if group2 is None:
            cell_idx2 = ~cell_idx1
        else:
            cell_idx2 = (adata.obs[groupby] == group2).to_numpy().ravel()

        all_info = dc.get_bayes_factors(
            cell_idx1,
            cell_idx2,
            mode=mode,
            delta=delta,
            batchid1=batchid1,
            batchid2=batchid2,
            use_observed_batches=not batch_correction,
            **kwargs,
        )

        if all_stats is True:
            genes_properties_dict = all_stats_fn(adata, cell_idx1, cell_idx2)
            all_info = {**all_info, **genes_properties_dict}

        res = pd.DataFrame(all_info, index=col_names)
        sort_key = "proba_de" if mode == "change" else "bayes_factor"
        res = res.sort_values(by=sort_key, ascending=False)
        if mode == "change":
            res["is_de_fdr_{}".format(fdr)] = _fdr_de_prediction(
                res["proba_de"], fdr=fdr
            )
        if idx1 is None:
            g2 = "Rest" if group2 is None else group2
            res["comparison"] = "{} vs {}".format(g1, g2)
        df_results.append(res)

    if temp_key is not None:
        del adata.obs[temp_key]

    result = pd.concat(df_results, axis=0)

    return result

def _fdr_de_prediction(posterior_probas: np.ndarray, fdr: float = 0.05):
    """Compute posterior expected FDR and tag features as DE."""
    if not posterior_probas.ndim == 1:
        raise ValueError("posterior_probas should be 1-dimensional")
    sorted_genes = np.argsort(-posterior_probas)
    sorted_pgs = posterior_probas[sorted_genes]
    cumulative_fdr = (1.0 - sorted_pgs).cumsum() / (1.0 + np.arange(len(sorted_pgs)))
    d = (cumulative_fdr <= fdr).sum()
    pred_de_genes = sorted_genes[:d]
    is_pred_de = np.zeros_like(cumulative_fdr).astype(bool)
    is_pred_de[pred_de_genes] = True
    return is_pred_de
<|endoftext|>