<commit_before>from django import forms
from django.core.validators import MinLengthValidator
from django.template.loader import render_to_string
from django.utils.translation import gettext
from django.utils.translation import gettext_lazy as _
from django.urls import reverse
from memoized import memoized

from crispy_forms import layout as crispy
from crispy_forms.helper import FormHelper
from crispy_forms.bootstrap import StrictButton

import langcodes
from corehq.apps.hqwebapp.crispy import FormActions, HQFormHelper, LinkButton
from corehq.apps.hqwebapp.fields import MultiEmailField
from corehq.apps.hqwebapp.widgets import SelectToggle
from corehq.apps.reports.models import TableauServer, TableauVisualization
from corehq.apps.saved_reports.models import (
    DEFAULT_REPORT_NOTIF_SUBJECT,
    ReportConfig,
    ReportNotification,
)
from corehq.apps.userreports.reports.view import ConfigurableReportView
from corehq.toggles import HOURLY_SCHEDULED_REPORT, NAMESPACE_DOMAIN

class SavedReportConfigForm(forms.Form):
    name = forms.CharField()
    description = forms.CharField(
        required=False,
        widget=forms.Textarea(attrs={"class": "vertical-resize"}),
    )
    start_date = forms.DateField(
        required=False,
    )
    end_date = forms.DateField(
        required=False,
    )
    datespan_slug = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    date_range = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    days = forms.IntegerField(
        required=False,
        widget=forms.HiddenInput(),
    )
    _id = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    report_slug = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    report_type = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    subreport_slug = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )

    def __init__(self, domain, user_id, *args, **kwargs):
        self.domain = domain
        self.user_id = user_id
        super(SavedReportConfigForm, self).__init__(*args, **kwargs)

    def clean(self):
        name = self.cleaned_data['name']
        _id = self.cleaned_data['_id']

        user_configs = ReportConfig.by_domain_and_owner(self.domain, self.user_id)
        if not _id and name in [c.name for c in user_configs]:
            raise forms.ValidationError(
                "A saved report with the name '%(name)s' already exists." % {
                    'name': name,
                }
            )

        date_range = self.cleaned_data['date_range']

        if (
            self.cleaned_data['report_type'] == ConfigurableReportView.prefix
            and not self.cleaned_data['datespan_slug']
        ):
            self.cleaned_data['date_range'] = None
        else:
            if date_range == 'last7':
                self.cleaned_data['days'] = 7
            elif date_range == 'last30':
                self.cleaned_data['days'] = 30
            elif (date_range == 'lastn' and self.cleaned_data.get('days') is None
                  and self.cleaned_data['report_type'] != ConfigurableReportView.prefix):
                raise forms.ValidationError(
                    "Field 'days' was expected but not provided."
                )

        return self.cleaned_data

class ScheduledReportForm(forms.Form):
    INTERVAL_CHOICES = [
        ("daily", _("Daily")),
        ("weekly", _("Weekly")),
        ("monthly", _("Monthly"))
    ]

    config_ids = forms.MultipleChoiceField(
        label=_("Saved report(s)"),
        validators=[MinLengthValidator(1)],
        help_text='Note: not all built-in reports support email delivery, so'
                  ' some of your saved reports may not appear in this list')

    interval = forms.TypedChoiceField(
        label=_('Interval'),
        widget=SelectToggle(choices=INTERVAL_CHOICES, apply_bindings=True),
        choices=INTERVAL_CHOICES)

    day = forms.TypedChoiceField(
        label=_("Day"),
        coerce=int,
        required=False,
        choices=[(i, i) for i in range(0, 32)])

    hour = forms.TypedChoiceField(
        label=_('Time'),
        coerce=int,
        choices=ReportNotification.hour_choices())

    stop_hour = forms.TypedChoiceField(
        label=_('To Time'),
        coerce=int,
        choices=ReportNotification.hour_choices())

    start_date = forms.DateField(
        label=_('Report Start Date'),
        required=False
    )

    send_to_owner = forms.BooleanField(
        label=_('Send to owner'),
        required=False
    )

    attach_excel = forms.BooleanField(
        label=_('Attach Excel Report'),
        required=False
    )

    recipient_emails = MultiEmailField(
        label=_('Other recipients'),
        required=False
    )
    email_subject = forms.CharField(
        required=False,
        help_text='Translated into recipient\'s language if set to "%(default_subject)s".' % {
            'default_subject': DEFAULT_REPORT_NOTIF_SUBJECT,
        },
    )

    language = forms.ChoiceField(
        label=_('Language'),
        required=False,
        choices=[('', '')] + langcodes.get_all_langs_for_select(),
        widget=forms.Select()
    )

    def __init__(self, *args, **kwargs):
        super(ScheduledReportForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper()
        self.helper.form_class = 'form-horizontal'
        self.helper.form_id = 'id-scheduledReportForm'
        self.helper.label_class = 'col-sm-3 col-md-2'
        self.helper.field_class = 'col-sm-9 col-md-8 col-lg-6'

        domain = kwargs.get('initial', {}).get('domain', None)
        if domain is not None and HOURLY_SCHEDULED_REPORT.enabled(domain, NAMESPACE_DOMAIN):
            self.fields['interval'].choices.insert(0, ("hourly", gettext("Hourly")))
            self.fields['interval'].widget.choices.insert(0, ("hourly", gettext("Hourly")))

        self.helper.add_layout(
            crispy.Layout(
                crispy.Fieldset(
                    gettext("Configure Scheduled Report"),
                    'config_ids',
                    'interval',
                    'day',
                    'hour',
                    'stop_hour',
                    'start_date',
                    crispy.Field(
                        'email_subject',
                        css_class='input-xlarge',
                    ),
                    crispy.Field(
                        'send_to_owner'
                    ),
                    crispy.Field(
                        'attach_excel'
                    ),
                    'recipient_emails',
                    'language',
                    crispy.HTML(
                        render_to_string('reports/partials/privacy_disclaimer.html')
                    )
                ),
                FormActions(
                    crispy.Submit('submit_btn', 'Submit')
                )
            )
        )

    def clean(self):
        cleaned_data = super(ScheduledReportForm, self).clean()
        if cleaned_data.get("interval") == "daily":
            del cleaned_data["day"]
        if cleaned_data.get("interval") == "hourly":
            del cleaned_data["day"]
        _verify_email(cleaned_data)
        return cleaned_data

    def clean_stop_hour(self):
        cleaned_data = super(ScheduledReportForm, self).clean()
        if cleaned_data.get("interval") == "hourly":
            if cleaned_data['hour'] > cleaned_data['stop_hour']:
                self.add_error('stop_hour', _("Must be after 'From Time'"))

        return cleaned_data.get('stop_hour')

class EmailReportForm(forms.Form):
    subject = forms.CharField(required=False)
    send_to_owner = forms.BooleanField(required=False)
    attach_excel = forms.BooleanField(required=False)
    recipient_emails = MultiEmailField(required=False)
    notes = forms.CharField(required=False)

    def clean(self):
        cleaned_data = super(EmailReportForm, self).clean()
        _verify_email(cleaned_data)
        return cleaned_data

def _verify_email(cleaned_data):
    if ('recipient_emails' in cleaned_data
        and not (cleaned_data['recipient_emails'] or
                     cleaned_data['send_to_owner'])):
        raise forms.ValidationError("You must specify at least one "
                                    "valid recipient")

class TableauServerForm(forms.Form):

    server_type = forms.CharField(
        label=_('Server Type'),
        widget=forms.Select(choices=[
            ("", _("Select server type")),
            ('server', _('Tableau Server')),
            ('online', _('Tableau Online')),
        ]),
    )

    server_name = forms.CharField(
        label=_('Server Name')
    )

    validate_hostname = forms.CharField(
        label=_('Validate Hostname'),
        required=False,
    )

    target_site = forms.CharField(
        label=_('Target Site'),
    )

    class Meta:
        model = TableauServer
        fields = [
            'server_type',
            'server_name',
            'validate_hostname',
            'target_site',
        ]

    def __init__(self, data, *args, **kwargs):
        self.domain = kwargs.pop('domain')
        kwargs['initial'] = self.initial_data
        super(TableauServerForm, self).__init__(data, *args, **kwargs)

        self.helper = HQFormHelper()
        self.helper.form_method = 'POST'
        self.helper.layout = crispy.Layout(
            crispy.Div(
                crispy.Field('server_type'),
            ),
            crispy.Div(
                crispy.Field('server_name'),
            ),
            crispy.Div(
                crispy.Field('validate_hostname'),
            ),
            crispy.Div(
                crispy.Field('target_site'),
            ),
            FormActions(
                crispy.Submit('submit_btn', 'Submit')
            )
        )

    @property
    @memoized
    def _existing_config(self):
        existing, _created = TableauServer.objects.get_or_create(
            domain=self.domain
        )
        return existing

    @property
    def initial_data(self):
        return {
            'server_type': self._existing_config.server_type,
            'server_name': self._existing_config.server_name,
            'validate_hostname': self._existing_config.validate_hostname,
            'target_site': self._existing_config.target_site,
        }

    def save(self):
        self._existing_config.server_type = self.cleaned_data['server_type']
        self._existing_config.server_name = self.cleaned_data['server_name']
        self._existing_config.validate_hostname = self.cleaned_data['validate_hostname']
        self._existing_config.target_site = self.cleaned_data['target_site']
        self._existing_config.save()

class TableauVisualizationForm(forms.ModelForm):
    view_url = forms.CharField(
        label=_('View URL'),
    )

    class Meta:
        model = TableauVisualization
        fields = [
            'title',
            'server',
            'view_url',
        ]

    def __init__(self, domain, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.domain = domain
        self.fields['server'].queryset = TableauServer.objects.filter(domain=domain)

    @property
    def helper(self):
        helper = HQFormHelper()
        from corehq.apps.reports.views import TableauVisualizationListView
        helper.layout = crispy.Layout(
            crispy.Field('title'),
            crispy.Field('server'),
            crispy.Field('view_url'),

            FormActions(
                StrictButton(
                    _("Save"),
                    type="submit",
                    css_class="btn btn-primary",
                ),
                LinkButton(
                    _("Cancel"),
                    reverse(
                        TableauVisualizationListView.urlname,
                        kwargs={'domain': self.domain},
                    ),
                    css_class="btn btn-default",
                ),
            ),
        )
        return helper

    def save(self, commit=True):
        self.instance.domain = self.domain
        return super().save(commit)

class UpdateTableauVisualizationForm(TableauVisualizationForm):
    pass
<commit_msg>feat: creates form to update tableau visualization<commit_after>from django import forms
from django.core.validators import MinLengthValidator
from django.template.loader import render_to_string
from django.utils.translation import gettext
from django.utils.translation import gettext_lazy as _
from django.urls import reverse
from memoized import memoized

from crispy_forms import layout as crispy
from crispy_forms.helper import FormHelper
from crispy_forms.bootstrap import StrictButton

import langcodes
from corehq.apps.hqwebapp.crispy import FormActions, HQFormHelper, LinkButton
from corehq.apps.hqwebapp.fields import MultiEmailField
from corehq.apps.hqwebapp.widgets import SelectToggle
from corehq.apps.reports.models import TableauServer, TableauVisualization
from corehq.apps.saved_reports.models import (
    DEFAULT_REPORT_NOTIF_SUBJECT,
    ReportConfig,
    ReportNotification,
)
from corehq.apps.userreports.reports.view import ConfigurableReportView
from corehq.toggles import HOURLY_SCHEDULED_REPORT, NAMESPACE_DOMAIN

class SavedReportConfigForm(forms.Form):
    name = forms.CharField()
    description = forms.CharField(
        required=False,
        widget=forms.Textarea(attrs={"class": "vertical-resize"}),
    )
    start_date = forms.DateField(
        required=False,
    )
    end_date = forms.DateField(
        required=False,
    )
    datespan_slug = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    date_range = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    days = forms.IntegerField(
        required=False,
        widget=forms.HiddenInput(),
    )
    _id = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    report_slug = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    report_type = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )
    subreport_slug = forms.CharField(
        required=False,
        widget=forms.HiddenInput(),
    )

    def __init__(self, domain, user_id, *args, **kwargs):
        self.domain = domain
        self.user_id = user_id
        super(SavedReportConfigForm, self).__init__(*args, **kwargs)

    def clean(self):
        name = self.cleaned_data['name']
        _id = self.cleaned_data['_id']

        user_configs = ReportConfig.by_domain_and_owner(self.domain, self.user_id)
        if not _id and name in [c.name for c in user_configs]:
            raise forms.ValidationError(
                "A saved report with the name '%(name)s' already exists." % {
                    'name': name,
                }
            )

        date_range = self.cleaned_data['date_range']

        if (
            self.cleaned_data['report_type'] == ConfigurableReportView.prefix
            and not self.cleaned_data['datespan_slug']
        ):
            self.cleaned_data['date_range'] = None
        else:
            if date_range == 'last7':
                self.cleaned_data['days'] = 7
            elif date_range == 'last30':
                self.cleaned_data['days'] = 30
            elif (date_range == 'lastn' and self.cleaned_data.get('days') is None
                  and self.cleaned_data['report_type'] != ConfigurableReportView.prefix):
                raise forms.ValidationError(
                    "Field 'days' was expected but not provided."
                )

        return self.cleaned_data

class ScheduledReportForm(forms.Form):
    INTERVAL_CHOICES = [
        ("daily", _("Daily")),
        ("weekly", _("Weekly")),
        ("monthly", _("Monthly"))
    ]

    config_ids = forms.MultipleChoiceField(
        label=_("Saved report(s)"),
        validators=[MinLengthValidator(1)],
        help_text='Note: not all built-in reports support email delivery, so'
                  ' some of your saved reports may not appear in this list')

    interval = forms.TypedChoiceField(
        label=_('Interval'),
        widget=SelectToggle(choices=INTERVAL_CHOICES, apply_bindings=True),
        choices=INTERVAL_CHOICES)

    day = forms.TypedChoiceField(
        label=_("Day"),
        coerce=int,
        required=False,
        choices=[(i, i) for i in range(0, 32)])

    hour = forms.TypedChoiceField(
        label=_('Time'),
        coerce=int,
        choices=ReportNotification.hour_choices())

    stop_hour = forms.TypedChoiceField(
        label=_('To Time'),
        coerce=int,
        choices=ReportNotification.hour_choices())

    start_date = forms.DateField(
        label=_('Report Start Date'),
        required=False
    )

    send_to_owner = forms.BooleanField(
        label=_('Send to owner'),
        required=False
    )

    attach_excel = forms.BooleanField(
        label=_('Attach Excel Report'),
        required=False
    )

    recipient_emails = MultiEmailField(
        label=_('Other recipients'),
        required=False
    )
    email_subject = forms.CharField(
        required=False,
        help_text='Translated into recipient\'s language if set to "%(default_subject)s".' % {
            'default_subject': DEFAULT_REPORT_NOTIF_SUBJECT,
        },
    )

    language = forms.ChoiceField(
        label=_('Language'),
        required=False,
        choices=[('', '')] + langcodes.get_all_langs_for_select(),
        widget=forms.Select()
    )

    def __init__(self, *args, **kwargs):
        super(ScheduledReportForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper()
        self.helper.form_class = 'form-horizontal'
        self.helper.form_id = 'id-scheduledReportForm'
        self.helper.label_class = 'col-sm-3 col-md-2'
        self.helper.field_class = 'col-sm-9 col-md-8 col-lg-6'

        domain = kwargs.get('initial', {}).get('domain', None)
        if domain is not None and HOURLY_SCHEDULED_REPORT.enabled(domain, NAMESPACE_DOMAIN):
            self.fields['interval'].choices.insert(0, ("hourly", gettext("Hourly")))
            self.fields['interval'].widget.choices.insert(0, ("hourly", gettext("Hourly")))

        self.helper.add_layout(
            crispy.Layout(
                crispy.Fieldset(
                    gettext("Configure Scheduled Report"),
                    'config_ids',
                    'interval',
                    'day',
                    'hour',
                    'stop_hour',
                    'start_date',
                    crispy.Field(
                        'email_subject',
                        css_class='input-xlarge',
                    ),
                    crispy.Field(
                        'send_to_owner'
                    ),
                    crispy.Field(
                        'attach_excel'
                    ),
                    'recipient_emails',
                    'language',
                    crispy.HTML(
                        render_to_string('reports/partials/privacy_disclaimer.html')
                    )
                ),
                FormActions(
                    crispy.Submit('submit_btn', 'Submit')
                )
            )
        )

    def clean(self):
        cleaned_data = super(ScheduledReportForm, self).clean()
        if cleaned_data.get("interval") == "daily":
            del cleaned_data["day"]
        if cleaned_data.get("interval") == "hourly":
            del cleaned_data["day"]
        _verify_email(cleaned_data)
        return cleaned_data

    def clean_stop_hour(self):
        cleaned_data = super(ScheduledReportForm, self).clean()
        if cleaned_data.get("interval") == "hourly":
            if cleaned_data['hour'] > cleaned_data['stop_hour']:
                self.add_error('stop_hour', _("Must be after 'From Time'"))

        return cleaned_data.get('stop_hour')

class EmailReportForm(forms.Form):
    subject = forms.CharField(required=False)
    send_to_owner = forms.BooleanField(required=False)
    attach_excel = forms.BooleanField(required=False)
    recipient_emails = MultiEmailField(required=False)
    notes = forms.CharField(required=False)

    def clean(self):
        cleaned_data = super(EmailReportForm, self).clean()
        _verify_email(cleaned_data)
        return cleaned_data

def _verify_email(cleaned_data):
    if ('recipient_emails' in cleaned_data
        and not (cleaned_data['recipient_emails'] or
                     cleaned_data['send_to_owner'])):
        raise forms.ValidationError("You must specify at least one "
                                    "valid recipient")

class TableauServerForm(forms.Form):

    server_type = forms.CharField(
        label=_('Server Type'),
        widget=forms.Select(choices=[
            ("", _("Select server type")),
            ('server', _('Tableau Server')),
            ('online', _('Tableau Online')),
        ]),
    )

    server_name = forms.CharField(
        label=_('Server Name')
    )

    validate_hostname = forms.CharField(
        label=_('Validate Hostname'),
        required=False,
    )

    target_site = forms.CharField(
        label=_('Target Site'),
    )

    class Meta:
        model = TableauServer
        fields = [
            'server_type',
            'server_name',
            'validate_hostname',
            'target_site',
        ]

    def __init__(self, data, *args, **kwargs):
        self.domain = kwargs.pop('domain')
        kwargs['initial'] = self.initial_data
        super(TableauServerForm, self).__init__(data, *args, **kwargs)

        self.helper = HQFormHelper()
        self.helper.form_method = 'POST'
        self.helper.layout = crispy.Layout(
            crispy.Div(
                crispy.Field('server_type'),
            ),
            crispy.Div(
                crispy.Field('server_name'),
            ),
            crispy.Div(
                crispy.Field('validate_hostname'),
            ),
            crispy.Div(
                crispy.Field('target_site'),
            ),
            FormActions(
                crispy.Submit('submit_btn', 'Submit')
            )
        )

    @property
    @memoized
    def _existing_config(self):
        existing, _created = TableauServer.objects.get_or_create(
            domain=self.domain
        )
        return existing

    @property
    def initial_data(self):
        return {
            'server_type': self._existing_config.server_type,
            'server_name': self._existing_config.server_name,
            'validate_hostname': self._existing_config.validate_hostname,
            'target_site': self._existing_config.target_site,
        }

    def save(self):
        self._existing_config.server_type = self.cleaned_data['server_type']
        self._existing_config.server_name = self.cleaned_data['server_name']
        self._existing_config.validate_hostname = self.cleaned_data['validate_hostname']
        self._existing_config.target_site = self.cleaned_data['target_site']
        self._existing_config.save()

class TableauVisualizationForm(forms.ModelForm):
    view_url = forms.CharField(
        label=_('View URL'),
    )

    class Meta:
        model = TableauVisualization
        fields = [
            'title',
            'server',
            'view_url',
        ]

    def __init__(self, domain, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.domain = domain
        self.fields['server'].queryset = TableauServer.objects.filter(domain=domain)

    @property
    def helper(self):
        helper = HQFormHelper()
        from corehq.apps.reports.views import TableauVisualizationListView
        helper.layout = crispy.Layout(
            crispy.Field('title'),
            crispy.Field('server'),
            crispy.Field('view_url'),

            FormActions(
                StrictButton(
                    _("Save"),
                    type="submit",
                    css_class="btn btn-primary",
                ),
                LinkButton(
                    _("Cancel"),
                    reverse(
                        TableauVisualizationListView.urlname,
                        kwargs={'domain': self.domain},
                    ),
                    css_class="btn btn-default",
                ),
            ),
        )
        return helper

    def save(self, commit=True):
        self.instance.domain = self.domain
        return super().save(commit)

class UpdateTableauVisualizationForm(TableauVisualizationForm):
    id = forms.CharField(widget=forms.HiddenInput())

    class Meta:
        model = TableauVisualization
        fields = [
            'id',
            'title',
            'server',
            'view_url',
        ]

    @property
    def helper(self):
        helper = HQFormHelper()
        helper.form_style = 'default'
        helper.form_show_labels = True
        helper.layout = crispy.Layout(
            crispy.Div(
                crispy.Field('id'),
                crispy.Field('title'),
                crispy.Field('server'),
                crispy.Field('view_url'),
                css_class='modal-body',
            ),
            FormActions(
                StrictButton(
                    _("Update"),
                    css_class='btn btn-primary',
                    type='submit',
                ),
                crispy.Button(
                    'cancel',
                    _("Cancel"),
                    css_class="btn btn-default",
                    data_dismiss="modal",
                ),
                css_class='modal-footer',
            ),
        )
        return helper
<|endoftext|>


<commit_before>#TODO: Update with new format
# ####################################
# 22/01/2013
# Luxembourg University SnT
# Project PROTECT
# Alejandro Correa Bahnsen
#####################################
# CALCULATE COST-SENSITIVE LOGISTIC REGRESSION
# MINIMIZING THE COST DUE TO FRAUDS
#
#####################################

from ..optimization import GAcont
from ..metrics import cost_measure

import numpy as np
import math
from scipy.optimize import minimize

# TODO: Fix CSLog function
class CSLogisticRegression():
    def __init__(self):
        def sigmoid2(t):
            if t < -10:
                return 0.00000001
            else:
                return 1.0 / (1 + math.exp(-t))

        self.sigmoid = np.vectorize(sigmoid2)
        self.fitnessfunc = cost
        return None

    # def fitnessfunc_i(theta, y, x, cost_mat, reg):
    #     def sigmoid(t):
    #         # TODO: FIX
    #         #missing to compare if this is faster than without vectorization, since it have to be calculated each time
    #         def sigmoid2(t):
    #             if t<-10:
    #                 return 0.00000001
    #             else:
    #                 return 1.0 / ( 1+ math.exp(-t))
    #         return np.vectorize(sigmoid2)(t)
    #     m = y.shape[0]
    #     p = (sigmoid(np.dot(x, theta.transpose()))).reshape(y.shape)
    #     # TODO: impact in cost by sending probabilities or class prediction
    #     cost_i = cost_measure(y, p, cost_mat)
    #     cost_i += reg / m * ((theta ** 2).sum())
    #     return cost_i
    #
    # fitnessfunc = np.vectorize(fitnessfunc_i)

    # TODO: Convert parameters to diccionary
    def fit(self, x, y, cost_mat, intercept=True, reg=0, method='BFGS', range1=None, params_ga=[100, 100, 10, 0.25]):
        #Function to fit a Logistic Regression
        #Input matrix X (not structure) and vector Y.
        #If jac!=False is because manual gradient is calculated
        #Otherwise gradient is calculated by the optimization function
        #On initial tests using the gradient make the algorithm run 2X faster

        setattr(self, 'intercept', intercept)
        if intercept == True:
            x = np.hstack((np.ones((x.shape[0], 1)), x))

        n = x.shape[1]
        initial_theta = np.zeros((n, 1))
        if method == "GA":
            if range1 == None:
                range1 = np.vstack((-10 * np.ones((1, n)), 10 * np.ones((1, n))))
            res = GAcont(n, self.fitnessfunc, params_ga[0], params_ga[1], CS=params_ga[2], MP=params_ga[3],
                         range=range1,
                         fargs=[y, x, cost_mat, reg])
            res.evaluate()
        else:
            res = minimize(self.fitnessfunc, initial_theta, (y, x, cost_mat, reg,), method=method,
                           options={'maxiter': 100, 'disp': True})

        setattr(self, 'theta', res.x)
        setattr(self, 'opti', res)
        setattr(self, 'hist', res.hist)
        setattr(self, 'full_hist', res.full_hist)

    def predict_proba(self, x_test):
        #Calculate the prediction of a LogRegression
        if self.intercept == True:
            x_test = np.hstack((np.ones((x_test.shape[0], 1)), x_test))
        p = np.zeros((x_test.shape[0], 2))
        p[:, 1] = self.sigmoid(np.dot(x_test, self.theta))
        p[:, 0] = 1 - p[:, 1]
        return p

    def predict(self, x_test, cut_point=0.5):
        #Calculate the prediction of a LogRegression
        p = np.floor(self.predict_proba(x_test)[:, 1] + (1 - cut_point))
        return p.reshape(p.shape[0], )

import numpy as np, math, pp, thread

#global init Job_server_prof
def sigmoid(t):
    #missing to compare if this is faster than without vectorization, since it have to be calculated each time
    def sigmoid2(t):
        if t < -10:
            return 0.00000001
        else:
            return 1.0 / ( 1 + math.exp(-t))

    return np.vectorize(sigmoid2)(t)

Job_server_prof = pp.Server(restart=True)

def cost(theta, y, x, cost_mat, reg, parallel=12):
    #theta_i is a vector 1xCols where x.shape[1]=Cols
    #theta is a matrix NxCols
    #check if theta=array
    def cost_i(j, j1, theta1, y, x, cost_mat, reg=0.0):
        res = np.empty(((min(j * j1 + j1, theta1.shape[0]) - j * j1), 2))
        m = y.shape[0]
        for i in range(j * j1, min(j * j1 + j1, theta1.shape[0])):
            # res[i-j*j1,1]=cost_p((sigmoid(np.dot(x,theta1[i].transpose()))).reshape(y.shape),y,amt,ca)/m
            p = (sigmoid(np.dot(x, theta1[i].transpose()))).reshape(y.shape)
            res[i - j * j1, 1] = cost_measure(y, p, cost_mat)
            res[i - j * j1, 1] + reg / m * ((theta1[i] ** 2).sum())
            res[i - j * j1, 0] = i
        return res

    class Res:
        def __init__(self, N):
            self.res = np.zeros(N, dtype='<f8')
            self.lock = thread.allocate_lock()

        def save_res(self, value):
            self.lock.acquire()
            for i in range(value.shape[0]):
                self.res[int(value[i, 0])] = value[i, 1]
            self.lock.release()

    if theta.shape[0] == theta.size:
        theta = theta.reshape(1, theta.shape[0])
    N = theta.shape[0]
    res = Res(N)
    n_jobs = min(parallel, N)
    j1 = int(math.ceil(float(N) / n_jobs))
    for j in range(n_jobs):
        Job_server_prof.submit(cost_i, (j, j1, theta, y, x, cost_mat, 0.0), (sigmoid, cost_measure,),
                               ('numpy as np', 'math'), callback=res.save_res)
    Job_server_prof.wait()
    return res.res
<commit_msg>cleaning regression<commit_after>#TODO: Update with new format
# ####################################
# 22/01/2013
# Luxembourg University SnT
# Project PROTECT
# Alejandro Correa Bahnsen
#####################################
# CALCULATE COST-SENSITIVE LOGISTIC REGRESSION
# MINIMIZING THE COST DUE TO FRAUDS
#
#####################################

from ..optimization import GAcont
from ..metrics import cost_measure

import numpy as np
import math
from scipy.optimize import minimize

# TODO: Fix CSLog function
class CSLogisticRegression():
    def __init__(self):
        def sigmoid2(t):
            if t < -10:
                return 0.00000001
            else:
                return 1.0 / (1 + math.exp(-t))

        self.sigmoid = np.vectorize(sigmoid2)
        self.fitnessfunc = cost
        return None

    # TODO: Convert parameters to diccionary
    def fit(self, x, y, cost_mat, intercept=True, reg=0, method='BFGS', range1=None, params_ga=[100, 100, 10, 0.25]):
        #Function to fit a Logistic Regression
        #Input matrix X (not structure) and vector Y.
        #If jac!=False is because manual gradient is calculated
        #Otherwise gradient is calculated by the optimization function
        #On initial tests using the gradient make the algorithm run 2X faster

        setattr(self, 'intercept', intercept)
        if intercept == True:
            x = np.hstack((np.ones((x.shape[0], 1)), x))

        n = x.shape[1]
        initial_theta = np.zeros((n, 1))
        if method == "GA":
            if range1 == None:
                range1 = np.vstack((-10 * np.ones((1, n)), 10 * np.ones((1, n))))
            res = GAcont(n, self.fitnessfunc, params_ga[0], params_ga[1], CS=params_ga[2], MP=params_ga[3],
                         range=range1,
                         fargs=[y, x, cost_mat, reg])
            res.evaluate()
        else:
            res = minimize(self.fitnessfunc, initial_theta, (y, x, cost_mat, reg,), method=method,
                           options={'maxiter': 100, 'disp': True})

        setattr(self, 'theta', res.x)
        setattr(self, 'opti', res)
        setattr(self, 'hist', res.hist)
        setattr(self, 'full_hist', res.full_hist)

    def predict_proba(self, x_test):
        #Calculate the prediction of a LogRegression
        if self.intercept == True:
            x_test = np.hstack((np.ones((x_test.shape[0], 1)), x_test))
        p = np.zeros((x_test.shape[0], 2))
        p[:, 1] = self.sigmoid(np.dot(x_test, self.theta))
        p[:, 0] = 1 - p[:, 1]
        return p

    def predict(self, x_test, cut_point=0.5):
        #Calculate the prediction of a LogRegression
        p = np.floor(self.predict_proba(x_test)[:, 1] + (1 - cut_point))
        return p.reshape(p.shape[0], )

import numpy as np, math, pp, thread

#global init Job_server_prof
def sigmoid(t):
    #missing to compare if this is faster than without vectorization, since it have to be calculated each time
    def sigmoid2(t):
        if t < -10:
            return 0.00000001
        else:
            return 1.0 / ( 1 + math.exp(-t))

    return np.vectorize(sigmoid2)(t)

Job_server_prof = pp.Server(restart=True)

def cost(theta, y, x, cost_mat, reg, parallel=12):
    #theta_i is a vector 1xCols where x.shape[1]=Cols
    #theta is a matrix NxCols
    #check if theta=array
    def cost_i(j, j1, theta1, y, x, cost_mat, reg=0.0):
        res = np.empty(((min(j * j1 + j1, theta1.shape[0]) - j * j1), 2))
        m = y.shape[0]
        for i in range(j * j1, min(j * j1 + j1, theta1.shape[0])):
            # res[i-j*j1,1]=cost_p((sigmoid(np.dot(x,theta1[i].transpose()))).reshape(y.shape),y,amt,ca)/m
            p = (sigmoid(np.dot(x, theta1[i].transpose()))).reshape(y.shape)
            res[i - j * j1, 1] = cost_measure(y, p, cost_mat)
            res[i - j * j1, 1] + reg / m * ((theta1[i] ** 2).sum())
            res[i - j * j1, 0] = i
        return res

    class Res:
        def __init__(self, N):
            self.res = np.zeros(N, dtype='<f8')
            self.lock = thread.allocate_lock()

        def save_res(self, value):
            self.lock.acquire()
            for i in range(value.shape[0]):
                self.res[int(value[i, 0])] = value[i, 1]
            self.lock.release()

    if theta.shape[0] == theta.size:
        theta = theta.reshape(1, theta.shape[0])
    N = theta.shape[0]
    res = Res(N)
    n_jobs = min(parallel, N)
    j1 = int(math.ceil(float(N) / n_jobs))
    for j in range(n_jobs):
        Job_server_prof.submit(cost_i, (j, j1, theta, y, x, cost_mat, 0.0), (sigmoid, cost_measure,),
                               ('numpy as np', 'math'), callback=res.save_res)
    Job_server_prof.wait()
    return res.res
<|endoftext|>


<commit_before>from aiohttp.web import Response

def get_all_todos():
    return Response(text='get_all_todos')

def create_todos():
    return Response(text='create_todos')

def update_todos():
    return Response(text='update_todos')

def remove_todos():
    return Response(text='remove_todos')

def get_todo():
    return Response(text='get_todo')

def update_todo():
    return Response(text='update_todo')

def remove_todo():
    return Response(text='remove_todo')
<commit_msg>fix handlers argument<commit_after>from aiohttp.web import Response

def get_all_todos(request):
    return Response(text='get_all_todos')

def create_todos(request):
    return Response(text='create_todos')

def update_todos(request):
    return Response(text='update_todos')

def remove_todos(request):
    return Response(text='remove_todos')

def get_todo(request):
    return Response(text='get_todo')

def update_todo(request):
    return Response(text='update_todo')

def remove_todo(request):
    return Response(text='remove_todo')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

'''
File: MongoSplitter.py
Author: NYU ITP Team
Description: Will calculate splits for a given collection/database
and store/return them in MongoSplit objects
'''
from pymongo import Connection, uri_parser
from MongoInputSplit import MongoInputSplit

import logging
import bson

from pymongo.uri_parser import (_partition,
                                _rpartition,
                                parse_userinfo,
                                split_hosts,
                                split_options,
                                parse_uri)

def calculate_splits(config):
    """reads config to find out what type of split to perform"""
    #pass
    uri = config.get("inputURI") if "inputURI" in config else "mongodb://localhost/test.in"

    #HACK -> make config align with this ^^
    config['inputURI'] = uri
    #/HACK

    #config.getInputURI()
    uri_info = uri_parser.parse_uri(uri)

    host = uri_info['nodelist'][0][0]
    port = uri_info['nodelist'][0][1]
    database_name = uri_info['database']
    collection_name = uri_info['collection']

    connection = Connection(uri)
    db = connection[database_name]
    stats = db.command("collstats", collection_name)

    isSharded = False if "sharded" not in stats else stats["sharded"]
    useShards = False #config.canReadSplitsFromShards()
    useChunks = False #config.isShardChunkedSplittingEnabled()
    slaveOk = True #config.canReadSplitsFromSecondary()

    logging.info(" Calculate Splits Code ... Use Shards? " , useShards , ", Use Chunks? " , useChunks , "; Collection Sharded? " , isSharded);

    if config.get("createInputSplits"):
        logging.info( "Creation of Input Splits is enabled." )
        if isSharded and (useShards or useChunks):
            if useShards and useChunks:
                logging.warn( "Combining 'use chunks' and 'read from shards directly' can have unexpected & erratic behavior in a live system due to chunk migrations. " );

            logging.info( "Sharding mode calculation entering." );
            return calculate_sharded_splits( config, useShards, useChunks, slaveOk, uri, mongo );

        else: # perfectly ok for sharded setups to run with a normally calculated split. May even be more efficient for some cases
            logging.info( "Using Unsharded Split mode (Calculating multiple splits though)" );
            return calculate_unsharded_splits( config, slaveOk, uri, collection_name );

    else:
        logging.info( "Creation of Input Splits is disabled; Non-Split mode calculation entering." );
        return calculate_single_split( config );

def calculate_unsharded_splits(config, slaveOk, uri, collection_name):
    """@todo: Docstring for calculate_unsharded_splits

    :returns: @todo

    """
    splits = [] #will return this list
    print "calculating unsharded splits"

    # TODO: pass these fields VV as parameters? (02/26/12, 11:30, AFlock)
    connection = Connection(uri)
    db = connection[config["db_name"]]
    coll = db[config.get('collection_name')]

    q = {} if not "query" in config else config.get("query")

    print 'in unsharded splits'
    #print 'query in MongoSplitter is ' + q
    #for key in q:
        #print 'am I printing the query?'
        #print q[key]

    #create command
    #command to split should look like this VV
    #SON([('splitVector', u'test.test_data'), ('maxChunkSize', 2), ('force', False), ('keyPattern', {'x': 1})])
    split_key = config.get('splitKey')
    split_size = config.get('splitSize')
    full_name  = coll.full_name
    logging.info("Calculating unsharded splits on collection %s with Split Key %s" % (full_name, split_key))
    logging.info("Max split size :: %sMB" % split_size)

    cmd = bson.son.SON()
    cmd["splitVector"]  = full_name
    cmd["maxChunkSize"] = split_size
    cmd["keyPattern"]   = split_key
    cmd["force"]        = True

    logging.debug("Issuing Command: %s" % cmd)
    print "Issuing Command: %s" % cmd
    data = db.command(cmd)
    print data

    #results look like this VV
    #{u'ok': 1.0, u'splitKeys': [{u'_id': ObjectId('4f49775348d9846c5e582b00')}, {u'_id': ObjectId('4f49775548d9846c5e58553b')}]}

    if data.get("err"):
        raise Exception(data.get("err"))
    elif data.get("ok") != 1.0:
        print data
        raise Exception("Unable to calculate splits")

    split_data = data.get('splitKeys')
    if not split_data:
        logging.warning("WARNING: No Input Splits were calculated by the split code.  Proceeding with a *single* split. Data may be too small, try lowering 'mongo.input.split_size'  if this is undesirable.")
    else:
        logging.info("Calculated %s splits" % len(split_data))

        # TODO: what is the query q parameter? (02/26/12, 12:18, AFlock)
        last_key = None
        for bound in split_data:
            splits.append(_split(config, q, last_key, bound))
            last_key = bound
        splits.append(_split(config, q, last_key, None))

    return [s.format_uri_with_query() for s in splits]

def _split(config=None, q={}, min=None, max=None):
    """@todo: Docstring for _split
    :returns: an actual MongoSplit object
    """
    print "_split being created"
    query = bson.son.SON()
    query["$query"]  = q

    if min:
        query["$min"] = min

    if max:
        query["$max"] = max

    logging.info("Assembled Query: " , query)

    return MongoInputSplit(
            config.get("inputURI"),
            config.get("inputKey"),
            query,
            config.get("fields"),
            config.get("sort"),
            config.get("limit", 0),
            config.get("skip", 0),
            config.get("timeout", True))

def calculate_single_split(config):
    pass

def calculate_sharded_splits(config, etc):
    """Worry about this after unsharded splits are doen

    :returns: @todo
    """
    pass

def fetch_splits_from_shards(config):
    """@todo: Docstring for fetch_splits_from_shards

    :returns: @todo
    """
    pass

def fetch_splits_via_chunks(config):
    """@todo: Docstring for fetch_splits_via_chunks

    :returns: @todo
    """
    pass

'''
def get_new_URI(original_URI, new_URI, slave_OK):
    """@todo: Docstring for get_new_URI

    :original_URI: @todo
    :etc: @todo
    :returns: a new Mongo_URI
    """

#    pass
    orig_URI_string = SCHEME_LEN
    server_end = -1
    server_start = 0

    """to find the last index of / in the original URI string """
    idx = orig_URI_string[::-1].find("/")
    if idx < 0:
        server_end = len(orig_URI_string)
    else:
        server_end = idx

    idx = orig_URI_string.find("@")

    if idx > 0:
        server_start = idx + 1

    sb = orig_URI_string
    sb.replace(orig_URI_string[server_start:server_end], new_URI)
    if slave_OK != null:
        if "?" in orig_URI_string:
            sb.append("&slaveok=").append(slave_OK)
        else
            sb.append("?slaveok=").append(slave_OK)

    ans = SCHEME + sb
    logging.debug("get_new_URI(): original " + original_URI + " new uri: " + ans )
    return ans
'''
<commit_msg>fix warnings, use getConnection, take out semicolons<commit_after>#!/usr/bin/env python
# encoding: utf-8

'''
File: MongoSplitter.py
Author: NYU ITP Team
Description: Will calculate splits for a given collection/database
and store/return them in MongoSplit objects
'''
from pymongo import Connection, uri_parser
from MongoInputSplit import MongoInputSplit
from mongoUtil import getCollection, getConnection

import logging
import bson

from pymongo.uri_parser import (_partition,
                                _rpartition,
                                parse_userinfo,
                                split_hosts,
                                split_options,
                                parse_uri)

def calculate_splits(config):
    """reads config to find out what type of split to perform"""
    #pass
    uri = config.get("inputURI") if "inputURI" in config else "mongodb://localhost/test.in"

    #HACK -> make config align with this ^^
    config['inputURI'] = uri
    #/HACK

    #config.getInputURI()
    uri_info = uri_parser.parse_uri(uri)

    host = uri_info['nodelist'][0][0]
    port = uri_info['nodelist'][0][1]
    database_name = uri_info['database']
    collection_name = uri_info['collection']

    connection = getConnection(uri)
    db = connection[database_name]
    stats = db.command("collstats", collection_name)

    isSharded = False if "sharded" not in stats else stats["sharded"]
    useShards = False #config.canReadSplitsFromShards()
    useChunks = False #config.isShardChunkedSplittingEnabled()
    slaveOk = True #config.canReadSplitsFromSecondary()

    logging.info(" Calculate Splits Code ... Use Shards? - %s\nUse Chunks? - %s\nCollection Sharded? - %s" % (useShards, useChunks, isSharded))

    if config.get("createInputSplits"):
        logging.info( "Creation of Input Splits is enabled." )
        if isSharded and (useShards or useChunks):
            if useShards and useChunks:
                logging.warn( "Combining 'use chunks' and 'read from shards directly' can have unexpected & erratic behavior in a live system due to chunk migrations. " )

            logging.info( "Sharding mode calculation entering." )
            return calculate_sharded_splits( config, useShards, useChunks, slaveOk, uri, mongo )

        else: # perfectly ok for sharded setups to run with a normally calculated split. May even be more efficient for some cases
            logging.info( "Using Unsharded Split mode (Calculating multiple splits though)" )
            return calculate_unsharded_splits( config, slaveOk, uri, collection_name )

    else:
        logging.info( "Creation of Input Splits is disabled; Non-Split mode calculation entering." );
        return calculate_single_split( config )

def calculate_unsharded_splits(config, slaveOk, uri, collection_name):
    """@todo: Docstring for calculate_unsharded_splits

    :returns: @todo

    """
    splits = [] #will return this list
    print "calculating unsharded splits"

    # TODO: pass these fields VV as parameters? (02/26/12, 11:30, AFlock)
    connection = getConnection(uri)
    db = connection[config["db_name"]]
    coll = db[config.get('collection_name')]

    q = {} if not "query" in config else config.get("query")

    print 'in unsharded splits'
    #print 'query in MongoSplitter is ' + q
    #for key in q:
        #print 'am I printing the query?'
        #print q[key]

    #create command
    #command to split should look like this VV
    #SON([('splitVector', u'test.test_data'), ('maxChunkSize', 2), ('force', False), ('keyPattern', {'x': 1})])
    split_key = config.get('splitKey')
    split_size = config.get('splitSize')
    full_name  = coll.full_name
    logging.info("Calculating unsharded splits on collection %s with Split Key %s" % (full_name, split_key))
    logging.info("Max split size :: %sMB" % split_size)

    cmd = bson.son.SON()
    cmd["splitVector"]  = full_name
    cmd["maxChunkSize"] = split_size
    cmd["keyPattern"]   = split_key
    cmd["force"]        = True

    logging.debug("Issuing Command: %s" % cmd)
    print "Issuing Command: %s" % cmd
    data = db.command(cmd)
    print data

    #results look like this VV
    #{u'ok': 1.0, u'splitKeys': [{u'_id': ObjectId('4f49775348d9846c5e582b00')}, {u'_id': ObjectId('4f49775548d9846c5e58553b')}]}

    if data.get("err"):
        raise Exception(data.get("err"))
    elif data.get("ok") != 1.0:
        print data
        raise Exception("Unable to calculate splits")

    split_data = data.get('splitKeys')
    if not split_data:
        logging.warning("WARNING: No Input Splits were calculated by the split code.  Proceeding with a *single* split. Data may be too small, try lowering 'mongo.input.split_size'  if this is undesirable.")
    else:
        logging.info("Calculated %s splits" % len(split_data))

        # TODO: what is the query q parameter? (02/26/12, 12:18, AFlock)
        last_key = None
        for bound in split_data:
            splits.append(_split(config, q, last_key, bound))
            last_key = bound
        splits.append(_split(config, q, last_key, None))

    return [s.format_uri_with_query() for s in splits]

def _split(config=None, q={}, min=None, max=None):
    """@todo: Docstring for _split
    :returns: an actual MongoSplit object
    """
    print "_split being created"
    query = bson.son.SON()
    query["$query"]  = q

    if min:
        query["$min"] = min

    if max:
        query["$max"] = max

    logging.info("Assembled Query: " , query)

    return MongoInputSplit(
            config.get("inputURI"),
            config.get("inputKey"),
            query,
            config.get("fields"),
            config.get("sort"),
            config.get("limit", 0),
            config.get("skip", 0),
            config.get("timeout", True))

def calculate_single_split(config):
    pass

def calculate_sharded_splits(config, etc):
    """Worry about this after unsharded splits are doen

    :returns: @todo
    """
    pass

def fetch_splits_from_shards(config):
    """@todo: Docstring for fetch_splits_from_shards

    :returns: @todo
    """
    pass

def fetch_splits_via_chunks(config):
    """@todo: Docstring for fetch_splits_via_chunks

    :returns: @todo
    """
    pass

'''
def get_new_URI(original_URI, new_URI, slave_OK):
    """@todo: Docstring for get_new_URI

    :original_URI: @todo
    :etc: @todo
    :returns: a new Mongo_URI
    """

#    pass
    orig_URI_string = SCHEME_LEN
    server_end = -1
    server_start = 0

    """to find the last index of / in the original URI string """
    idx = orig_URI_string[::-1].find("/")
    if idx < 0:
        server_end = len(orig_URI_string)
    else:
        server_end = idx

    idx = orig_URI_string.find("@")

    if idx > 0:
        server_start = idx + 1

    sb = orig_URI_string
    sb.replace(orig_URI_string[server_start:server_end], new_URI)
    if slave_OK != null:
        if "?" in orig_URI_string:
            sb.append("&slaveok=").append(slave_OK)
        else
            sb.append("?slaveok=").append(slave_OK)

    ans = SCHEME + sb
    logging.debug("get_new_URI(): original " + original_URI + " new uri: " + ans )
    return ans
'''
<|endoftext|>


<commit_before># Copyright 2016 Autodesk Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import yaml
import warnings

from pyccc import engines

from moldesign import utils
from . import compute

default_engine = None

FREE_COMPUTE_CANNON = 'cloudcomputecannon.bionano.autodesk.com:9000'

RUNNING_ON_WORKER = (os.environ.get('IS_PYCCC_JOB', '0') == '1')

COMPUTE_CONNECTION_WARNING = """
WARNING: Failed to connect to a computational engine - MDT won't be able to run anything outside of Python.

You can fix this either:
  1) interactively, in a notebook, by running `moldesign.configure()`, or,
  2) by modifying the configuration dictionary `moldesign.compute.config`, then
     running `moldesign.compute.reset_compute_engine()` to try again."""

# TODO: *write* configuration plus initial install default; sensible defaults

config = utils.DotDict()
""" dict: dictionary of parameters (read from user's moldesign.yml at startup)

This dictionary configures how MDT interacts with "computational engines" - the systems that run
jobs, usually using docker images.

Notes:
    Values in this dictionary can be configured at runtime; however, after changing them,
    you should update the compute configuration by running
    ``moldesign.compute.reset_compute_engine()``

Configuration is specified using the following keys:

Args:
    engine_type (str): The computational job engine to use. Default: 'ccc'. Currently supported:
        'docker', 'ccc', or 'subprocess'
    default_repository (str): Repository to pull MDT's standard images from.
        default: 'docker.io/autodesk/moldesign:'
    default_python_image (str): Image to run python commands in
        (default: ``docker.io/autodesk/moldesign:moldesign_complete-[VERSION]``, where
        [VERSION] is the version of MDT)
    default_version_tag (str): Default version tag for docker images
         (default: ``moldesign.__version__``)
    default_docker_machine (str): Name of the docker machine to connect to; if
        ``engine_type=='docker'``, EITHER this OR ``default_docker_url`` (but not both) must
        be set. (default: 'default')
    default_docker_url (str): URL for the docker daemon to run; if ``engine_type=='docker'``,
        EITHER this OR ``default_docker_machine`` (but not both) must be set.
        (default: ``unix://var/run/docker.sock``, the URL for a local docker instance)

MDT uses a non-standard docker tagging system to store its docker images. Generally,
a given image is pulled from a URL of the form:
   ``http://[some_docker_registry/orgname/reponame]:[imagename]-[versiontag]``

For instance, the ambertools 0.6 image is stored in the default repository at:
   ``http://docker.io/autodesk/moldesign:ambertools-0.6``
"""

ENVVAR = "MOLDESIGN_CONFIG"
""" str: name of environmental variable that stores path to moldesign.yml file.

If this variable is not set, ``$HOME/.moldesign/moldesign.yml`` will be used by default."""

DEFAULT_CONFIG_PATH = os.path.join(os.environ['HOME'], '.moldesign/moldesign.yml')
""" str: default search path for moldesign.yml."""

CONFIG_DEFAULTS = utils.DotDict(engine_type='ccc',
                                default_repository='docker.io/autodesk/moldesign:',
                                default_ccc_host=FREE_COMPUTE_CANNON,
                                default_python_image=None,
                                default_docker_host='unix://var/run/docker.sock',
                                default_docker_machine='default',
                                default_version_tag='0.7.4a2')

DEF_CONFIG = CONFIG_DEFAULTS.copy()
""" dict: default configuration to be written to moldesign.yml if it doesn't exist
"""
for x in 'default_docker_machine'.split(): DEF_CONFIG.pop(x)

def registry_login(client, login):
    print 'Logging into docker registry @ %s ...' % login['registry'],
    sys.stdout.flush()
    try:
        client.login(**login)
    except Exception as exc:
        warnings.warn('Failed to connect to %s. ' % login['registry'] +
                      'Some functions may time out.')
    else:
        print 'done'

def write_config(path=None):
    if path is None:
        path = _get_config_path()

    if not os.path.exists(path) and path == DEFAULT_CONFIG_PATH:
        confdir = os.path.dirname(path)
        if not os.path.exists(confdir):
            os.mkdir(confdir)
            print 'Created moldesign configuration directory %s' % confdir

    with open(path, 'w') as f:
        yaml.dump(config, f)

    print 'Wrote moldesign configuration to %s' % path

def get_engine():
    from moldesign import compute
    if compute.default_engine is None:
        try:
            reset_compute_engine()
        except:
            print COMPUTE_CONNECTION_WARNING
            raise
    return compute.default_engine

def init_config():
    """Called at the end of package import to read initial configuration and setup cloud computing.
    """
    config.update(CONFIG_DEFAULTS)

    path = _get_config_path()

    if os.path.exists(path):
        with open(path, 'r') as infile:
            print 'Reading configuration from %s' % path
            config.update(yaml.load(infile))

    if config.default_python_image is None:
        config.default_python_image = compute.get_image_path('moldesign_complete')

def _get_config_path():
    if ENVVAR in os.environ:
        path = os.environ[ENVVAR]
    else:
        path = DEFAULT_CONFIG_PATH
    return path

def reset_compute_engine():
    """Read the configuration dict at ``moldesign.compute.config`` and set up compute engine

    Sets the module-level variable ``default_engine``.

    Returns:
        dict: copy of the config dictionary used to set the engine
    """
    from moldesign import compute

    compute.default_engine = None

    if config.engine_type == 'docker-machine':
        with utils.textnotify('Connecting to docker-machine "%s"' % config.default_docker_machine):
            compute.default_engine = engines.DockerMachine(config.default_docker_machine)
        _connect_docker_registry()

    elif config.engine_type == 'docker':
        with utils.textnotify('Connecting to docker host at %s' % config.default_docker_host):
            compute.default_engine = engines.Docker(config.default_docker_host)
        _connect_docker_registry()

    elif config.engine_type == 'subprocess':
        compute.default_engine = engines.Subprocess()
        print """WARNING: running all computational jobs as subprocesses on this machine.
This requires that you have all necessary software installed locally.
To change the engine, call moldesign.configure() or modify moldesign.compute.config ."""

    elif config.engine_type in ('ccc', 'cloudcomputecannon'):
        with utils.textnotify('Connecting to CloudComputeCannon host at %s'
                              % config.default_ccc_host):
            compute.default_engine = engines.CloudComputeCannon(config.default_ccc_host)
            compute.default_engine.test_connection()

    else:
        raise ValueError('Unrecognized engine %s' % config.engine_type)

def _connect_docker_registry():
    from moldesign import compute

    if config.get('docker_registry_login', None):
        registry_login(compute.default_engine.client, config.docker_registry_login)
<commit_msg>Remove CCC and docker-machine support<commit_after># Copyright 2016 Autodesk Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import yaml
import warnings

from pyccc import engines

from moldesign import utils
from . import compute

default_engine = None

RUNNING_ON_WORKER = (os.environ.get('IS_PYCCC_JOB', '0') == '1')

COMPUTE_CONNECTION_WARNING = """
WARNING: Failed to connect to a computational engine - MDT won't be able to run
anything outside of Python.

You can fix this either:
  1) interactively, in a notebook, by running `moldesign.configure()`, or,
  2) by modifying the configuration dictionary `moldesign.compute.config`, then
     running `moldesign.compute.reset_compute_engine()` to try again."""

config = utils.DotDict()
""" dict: dictionary of parameters (read from user's moldesign.yml at startup)

This dictionary configures how MDT interacts with "computational engines" - the systems that run
jobs, usually using docker images.

Notes:
    Values in this dictionary can be configured at runtime; however, after changing them,
    you should update the compute configuration by running
    ``moldesign.compute.reset_compute_engine()``

Configuration is specified using the following keys:

Args:
    engine_type (str): The computational job engine to use. Currently supported:
        'docker' (default) or 'subprocess'
    default_repository (str): Repository to pull MDT's standard images from.
        default: 'docker.io/autodesk/moldesign:'
    default_python_image (str): Image to run python commands in
        (default: ``docker.io/autodesk/moldesign:moldesign_complete-[VERSION]``, where
        [VERSION] is the version of MDT)
    default_version_tag (str): Default version tag for docker images
         (default: ``moldesign.__version__``)
    default_docker_url (str): URL for the docker daemon to run; if ``engine_type=='docker'``,
        EITHER this OR ``default_docker_machine`` (but not both) must be set.
        (default: ``unix://var/run/docker.sock``, the URL for a local docker instance)

MDT uses a non-standard docker tagging system to store its docker images. Generally,
a given image is pulled from a URL of the form:
   ``http://[some_docker_registry/orgname/reponame]:[imagename]-[versiontag]``

For instance, the ambertools 0.6 image is stored in the default repository at:
   ``http://docker.io/autodesk/moldesign:ambertools-0.6``
"""

ENVVAR = "MOLDESIGN_CONFIG"
""" str: name of environmental variable that stores path to moldesign.yml file.

If this variable is not set, ``$HOME/.moldesign/moldesign.yml`` will be used by default."""

DEFAULT_CONFIG_PATH = os.path.join(os.environ['HOME'], '.moldesign/moldesign.yml')
""" str: default search path for moldesign.yml."""

CONFIG_DEFAULTS = utils.DotDict(engine_type='docker',
                                default_repository='docker.io/autodesk/moldesign:',
                                default_python_image=None,
                                default_docker_host='unix://var/run/docker.sock',
                                default_version_tag='0.7.4a2',
                                devmode=False)

DEF_CONFIG = CONFIG_DEFAULTS.copy()
""" dict: default configuration to be written to moldesign.yml if it doesn't exist
"""

def registry_login(client, login):
    print 'Logging into docker registry @ %s ...' % login['registry'],
    sys.stdout.flush()
    try:
        client.login(**login)
    except Exception as exc:
        warnings.warn('Failed to connect to %s. ' % login['registry'] +
                      'Some functions may time out.')
    else:
        print 'done'

def write_config(path=None):
    if path is None:
        path = _get_config_path()

    if not os.path.exists(path) and path == DEFAULT_CONFIG_PATH:
        confdir = os.path.dirname(path)
        if not os.path.exists(confdir):
            os.mkdir(confdir)
            print 'Created moldesign configuration directory %s' % confdir

    with open(path, 'w') as f:
        yaml.dump(config, f)

    print 'Wrote moldesign configuration to %s' % path

def get_engine():
    from moldesign import compute
    if compute.default_engine is None:
        try:
            reset_compute_engine()
        except:
            print COMPUTE_CONNECTION_WARNING
            raise
    return compute.default_engine

def init_config():
    """Called at the end of package import to read initial configuration and setup cloud computing.
    """
    config.update(CONFIG_DEFAULTS)

    path = _get_config_path()

    if os.path.exists(path):
        with open(path, 'r') as infile:
            print 'Reading configuration from %s' % path
            config.update(yaml.load(infile))

    if config.default_python_image is None:
        config.default_python_image = compute.get_image_path('moldesign_complete')

def _get_config_path():
    if ENVVAR in os.environ:
        path = os.environ[ENVVAR]
    else:
        path = DEFAULT_CONFIG_PATH
    return path

def reset_compute_engine():
    """Read the configuration dict at ``moldesign.compute.config`` and set up compute engine

    Sets the module-level variable ``default_engine``.

    Returns:
        dict: copy of the config dictionary used to set the engine
    """
    from moldesign import compute

    compute.default_engine = None

    if config.engine_type == 'docker':
        with utils.textnotify('Connecting to docker host at %s' % config.default_docker_host):
            compute.default_engine = engines.Docker(config.default_docker_host)
        _connect_docker_registry()

    elif config.engine_type == 'subprocess':
        compute.default_engine = engines.Subprocess()
        print """WARNING: running all computational jobs as subprocesses on this machine.
This requires that you have all necessary software installed locally.
To change the engine, call moldesign.configure() or modify moldesign.compute.config ."""

    elif config.engine_type in ('ccc', 'docker-machine'):
        raise ValueError('Computational engine type "%s" is no longer supported by MDT. '
                         'Please install docker (https://docker.io) and set: \n'
                         '   engine_type: docker'
                         'in ~/.moldesign/moldesign.yml')

    else:
        raise ValueError('Unrecognized engine %s' % config.engine_type)

def _connect_docker_registry():
    from moldesign import compute

    if config.get('docker_registry_login', None):
        registry_login(compute.default_engine.client, config.docker_registry_login)
<|endoftext|>


<commit_before>import json
import logging
import queue
import socket
import threading

"""
Exception thrown if the peer sends an error.
"""
class RpcException(Exception):
    pass

"""
Connects to a remote RPC peer.
"""
def connect(address, port, handler = {}, timeout=5):
    connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
    connection.settimeout(timeout)
    connection.connect((address, port))

    listener = Listener(connection, handler)
    return Proxy(listener)

"""
Object that acts as a stub for remote methods.

A proxy object is usually created by calling connect() to connect to a remote
peer.

The proxy also acts as a state object for the current connection, since the
object lasts as long as the connection does.
"""
class Proxy:
    """
    Creates a new proxy object around the given listener.
    """
    def __init__(self, listener):
        self.listener = listener
        self.listener.start()

    """
    Gets the name of the remote peer as a tuple of the address and port.
    """
    def get_peer_name(self):
        return self.listener.address

    """
    Calls a method on the peer.
    """
    def invoke(self, name, params = {}):
        request = {
            "jsonrpc": "2.0",
            "method": name,
            "params": params
        }

        # Send the request and then read the response.
        self.listener.send(request)
        response = self.listener.receive_and_wait()

        if "error" in response:
            raise RpcException(response["error"]["code"], response["error"]["message"])

        return response["result"]

    """
    Informs the peer that the connection is closing and then closes the socket.
    """
    def close(self):
        self.invoke("close")
        self.listener.close()

    """
    Provides syntax sugar for calling remote methods.
    """
    def __getattr__(self, name):
        def closure(**kwargs):
            return self.invoke(name, kwargs)

        return closure

"""
Wrapper around a socket that provides message parsing, writing, and background
reading.

Uses a dedicated thread for reading and handling incoming messages from the
remote peer.
"""
class Listener(threading.Thread):
    """
    Creates a new listener for a given socket.
    """
    def __init__(self, socket, handler, timeout = 1.0, bufsize = 4096):
        threading.Thread.__init__(self)

        self.message_queue = queue.Queue()
        self.open = False
        self.socket = socket
        self.handler = handler() # Instantiate the handler class
        self.bufsize = bufsize

        socket.settimeout(timeout)
        self.address = socket.getpeername()

    """
    Receives the next message if available, or raises an Empty exception.

    Returns the message as a dictionary.
    """
    def receive(self):
        # Attempt to get a message.
        message = self.message_queue.get(False)
        # If no exception was thrown, we can use the result.
        self.message_queue.task_done()
        return message

    """
    Receives the next message, waiting until it arrives.

    Returns the message as a dictionary.
    """
    def receive_and_wait(self):
        # Block until the item in the queue is accessible.
        message = self.message_queue.get(True)
        # Mark it as received.
        self.message_queue.task_done()
        return message

    """
    Sends a message to the peer and waits for the response.
    """
    def send(self, message):
        # Serialize the message.
        serialized = json.dumps(message)
        # Send a packet ending in a magic byte package separator.
        self.socket.sendall(serialized.encode() + b"\0\0\0\0")

    """
    Runs the listener, which reads from the peer forever until close() is called.
    """
    def run(self):
        # Set up a local byte buffer.
        buffer = bytearray()

        # Read forever until we are told to close.
        self.open = True
        while self.open:
            # Read some data from the peer when it arrives.
            try:
                packet = self.socket.recv(self.bufsize)
                buffer.extend(packet)
            except socket.timeout:
                continue
            except (ConnectionError, OSError):
                self.close()
                break

            if len(packet) == 0:
                self.close()
                break

            # Look for any complete messages in the buffer.
            while True:
                # Look for the "magic string" at the end of each message.
                pos = buffer.find(b"\0\0\0\0")
                if pos < 0:
                    break

                # Slice the correct portion out of the buffer.
                string = buffer[:pos].decode()
                del buffer[:pos + 4]

                # Parse the message as JSON.
                message = json.loads(string)
                logging.info("Received message from %s", self.address)

                if not "method" in message:
                    # If the message is a response, put it into the shared queue to alert consumers.
                    self.message_queue.put(message)
                else:
                    # The message is a request, so handle it now.
                    self._handle_request(message)

        # Make sure all consumer threads pick out their responses before we clean up.
        self.message_queue.join()

        # Make sure the socket gets closed if we were asked politely to close.
        self.close()

    """
    Closes the listener politely whenever the loop runs around next.
    """
    def close_later(self):
        self.open = False

    """
    Closes the listener immediately.
    """
    def close(self):
        self.open = False
        try:
            self.socket.shutdown(socket.SHUT_RDWR)
            self.socket.close()
        except OSError:
            pass

    def _handle_request(self, request):
        response = {
            "jsonrpc": "2.0"
        }

        # Attempt to invoke the requested method.
        try:
            # If the method is "close", close the connection.
            if request["method"] == "close":
                self.close_later()
                response["result"] = True

            # For any other method, invoke it on the handler.
            else:
                func = getattr(self.handler, request["method"])
                args = request["params"]
                return_val = func(**args)

                response["result"] = return_val
        except Exception as e:
            # Method errored, so respond with an error instead.
            response["error"] = {
                "code": 500,
                "message": str(e)
            }

        # Send back a response to the peer.
        self.send(response)
<commit_msg>Fix duplex messages in nested rpc calls<commit_after>import json
import logging
import queue
import socket
import threading

"""
Exception thrown if the peer sends an error.
"""
class RpcException(Exception):
    pass

"""
Connects to a remote RPC peer.
"""
def connect(address, port, handler, timeout=5):
    connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
    connection.settimeout(timeout)
    connection.connect((address, port))

    listener = Listener(connection)
    return Proxy(listener, handler)

"""
Object that acts as a stub for remote methods.

A proxy object is usually created by calling connect() to connect to a remote
peer.

The proxy also acts as a state object for the current connection, since the
object lasts as long as the connection does.
"""
class Proxy:
    """
    Creates a new proxy object around the given listener.
    """
    def __init__(self, listener, handler):
        self.listener = listener
        self.listener.handler = handler(self) # Instantiate the handler class
        self.listener.start()

    """
    Gets the name of the remote peer as a tuple of the address and port.
    """
    def get_peer_name(self):
        return self.listener.address

    """
    Calls a method on the peer.
    """
    def invoke(self, name, params = {}):
        request = {
            "jsonrpc": "2.0",
            "method": name,
            "params": params
        }

        # Send the request and then read the response.
        self.listener.send(request)
        response = self.listener.receive_and_wait()

        if "error" in response:
            raise RpcException(response["error"]["code"], response["error"]["message"])

        return response["result"]

    """
    Informs the peer that the connection is closing and then closes the socket.
    """
    def close(self):
        self.invoke("close")
        self.listener.close()

    """
    Provides syntax sugar for calling remote methods.
    """
    def __getattr__(self, name):
        def closure(**kwargs):
            return self.invoke(name, kwargs)

        return closure

"""
Wrapper around a socket that provides message parsing, writing, and background
reading.

Uses a dedicated thread for reading and handling incoming messages from the
remote peer.
"""
class Listener(threading.Thread):
    """
    Creates a new listener for a given socket.
    """
    def __init__(self, socket, timeout = 1.0, bufsize = 4096):
        threading.Thread.__init__(self)

        self.message_queue = queue.Queue()
        self.open = False
        self.socket = socket
        self.handler = None
        self.bufsize = bufsize

        socket.settimeout(timeout)
        self.address = socket.getpeername()

    """
    Receives the next message if available, or raises an Empty exception.

    Returns the message as a dictionary.
    """
    def receive(self):
        # Attempt to get a message.
        message = self.message_queue.get(False)
        # If no exception was thrown, we can use the result.
        self.message_queue.task_done()
        return message

    """
    Receives the next message, waiting until it arrives.

    Returns the message as a dictionary.
    """
    def receive_and_wait(self):
        # Block until the item in the queue is accessible.
        message = self.message_queue.get(True)
        # Mark it as received.
        self.message_queue.task_done()
        return message

    """
    Sends a message to the peer and waits for the response.
    """
    def send(self, message):
        # Serialize the message.
        serialized = json.dumps(message)
        # Send a packet ending in a magic byte package separator.
        self.socket.sendall(serialized.encode() + b"\0\0\0\0")

    """
    Runs the listener, which reads from the peer forever until close() is called.
    """
    def run(self):
        # Set up a local byte buffer.
        buffer = bytearray()

        # Read forever until we are told to close.
        self.open = True
        while self.open:
            # Read some data from the peer when it arrives.
            try:
                packet = self.socket.recv(self.bufsize)
                buffer.extend(packet)
            except socket.timeout:
                continue
            except (ConnectionError, OSError):
                self.close()
                break

            if len(packet) == 0:
                self.close()
                break

            # Look for any complete messages in the buffer.
            while True:
                # Look for the "magic string" at the end of each message.
                pos = buffer.find(b"\0\0\0\0")
                if pos < 0:
                    break

                # Slice the correct portion out of the buffer.
                string = buffer[:pos].decode()
                del buffer[:pos + 4]

                # Parse the message as JSON.
                message = json.loads(string)
                logging.info("Received message from %s", self.address)

                if not "method" in message:
                    # If the message is a response, put it into the shared queue to alert consumers.
                    self.message_queue.put(message)
                else:
                    # The message is a request, so handle it now.
                    threading.Thread(target=lambda: self._handle_request(message)).start()

        # Make sure all consumer threads pick out their responses before we clean up.
        self.message_queue.join()

        # Make sure the socket gets closed if we were asked politely to close.
        self.close()

    """
    Closes the listener politely whenever the loop runs around next.
    """
    def close_later(self):
        self.open = False

    """
    Closes the listener immediately.
    """
    def close(self):
        self.open = False
        try:
            self.socket.shutdown(socket.SHUT_RDWR)
            self.socket.close()
        except OSError:
            pass

    def _handle_request(self, request):
        response = {
            "jsonrpc": "2.0"
        }

        # Attempt to invoke the requested method.
        try:
            # If the method is "close", close the connection.
            if request["method"] == "close":
                self.close_later()
                response["result"] = True

            # For any other method, invoke it on the handler.
            else:
                func = getattr(self.handler, request["method"])
                args = request["params"]
                return_val = func(**args)

                response["result"] = return_val
        except Exception as e:
            # Method errored, so respond with an error instead.
            response["error"] = {
                "code": 500,
                "message": str(e)
            }

        # Send back a response to the peer.
        self.send(response)

"""
Base class for handlers.
"""
class Handler:
    def __init__(self, proxy):
        self.proxy = proxy
<|endoftext|>


<commit_before>from collections import OrderedDict

from django.utils.encoding import force_text
from django.utils.http import urlencode
from rest_framework import exceptions
from rest_framework.metadata import SimpleMetadata
from rest_framework.request import clone_request
from rest_framework.reverse import reverse

from nodeconductor.core.utils import sort_dict

class ActionSerializer(object):
    def __init__(self, func, name, request, view, resource):
        self.func = func
        self.name = name
        self.request = request
        self.resource = resource
        self.view = view

    def serialize(self):
        reason = self.get_reason()

        return {
            'title': self.get_title(),
            'method': self.get_method(),
            'destructive': self.is_destructive(),
            'url': self.get_url(),
            'reason': reason,
            'enabled': not reason
        }

    def is_destructive(self):
        if self.name == 'destroy':
            return True
        return getattr(self.func, 'destructive', False)

    def get_title(self):
        try:
            return getattr(self.func, 'title')
        except AttributeError:
            return self.name.replace('_', ' ').title()

    def get_reason(self):
        try:
            self.view.initial(self.request)
        except exceptions.APIException as e:
            return force_text(e)

    def get_method(self):
        if self.name == 'destroy':
            return 'DELETE'
        elif self.name == 'update':
            return 'PUT'
        return getattr(self.func, 'method', 'POST')

    def get_url(self):
        base_url = self.request.build_absolute_uri()
        method = self.get_method()
        if method in ('DELETE', 'PUT'):
            return base_url
        return base_url + self.name + '/'

class ActionsMetadata(SimpleMetadata):
    """
    Difference from SimpleMetadata class:
    1) Skip read-only fields, because options are used only for provisioning new resource.
    2) Don't expose choices for fields with queryset in order to reduce size of response.
    3) Attach actions metadata
    """
    def determine_metadata(self, request, view):
        self.request = request
        metadata = OrderedDict()
        if view.lookup_field in view.kwargs:
            metadata['actions'] = self.get_actions(request, view)
        else:
            metadata['actions'] = self.determine_actions(request, view)
        return metadata

    def get_actions(self, request, view):
        """
        Return metadata for resource-specific actions,
        such as start, stop, unlink
        """
        metadata = OrderedDict()
        actions = self.get_resource_actions(view)

        resource = view.get_object()
        for action_name, action in actions.items():
            if action_name == 'update':
                view.request = clone_request(request, 'PUT')
            else:
                view.action = action_name

            data = ActionSerializer(action, action_name, request, view, resource)
            metadata[action_name] = data.serialize()
            if not metadata[action_name]['enabled']:
                continue
            fields = self.get_action_fields(view, action_name, resource)
            if not fields:
                metadata[action_name]['type'] = 'button'
            else:
                metadata[action_name]['type'] = 'form'
                metadata[action_name]['fields'] = fields

            view.action = None
            view.request = request

        return metadata

    @classmethod
    def get_resource_actions(cls, view):
        actions = {}
        disabled_actions = getattr(view.__class__, 'disabled_actions', [])

        for key in dir(view.__class__):
            callback = getattr(view.__class__, key)
            if getattr(callback, 'deprecated', False):
                continue
            if 'post' not in getattr(callback, 'bind_to_methods', []):
                continue
            if key in disabled_actions:
                continue
            actions[key] = callback

        if 'DELETE' in view.allowed_methods and 'destroy' not in disabled_actions:
            actions['destroy'] = view.destroy

        if 'PUT' in view.allowed_methods and 'update' not in disabled_actions:
            actions['update'] = view.update

        return sort_dict(actions)

    def get_action_fields(self, view, action_name, resource):
        """
        Get fields exposed by action's serializer
        """
        serializer = view.get_serializer(resource)
        fields = OrderedDict()
        if not isinstance(serializer, view.serializer_class) or action_name == 'update':
            fields = self.get_fields(serializer.fields)
        return fields

    def get_serializer_info(self, serializer):
        """
        Given an instance of a serializer, return a dictionary of metadata
        about its fields.
        """
        if hasattr(serializer, 'child'):
            # If this is a `ListSerializer` then we want to examine the
            # underlying child serializer instance instead.
            serializer = serializer.child
        return self.get_fields(serializer.fields)

    def get_fields(self, serializer_fields):
        """
        Get fields metadata skipping empty fields
        """
        fields = OrderedDict()
        for field_name, field in serializer_fields.items():
            info = self.get_field_info(field, field_name)
            if info:
                fields[field_name] = info
        return fields

    def get_field_info(self, field, field_name):
        """
        Given an instance of a serializer field, return a dictionary
        of metadata about it.
        """
        field_info = OrderedDict()
        field_info['type'] = self.label_lookup[field]
        field_info['required'] = getattr(field, 'required', False)

        attrs = [
            'label', 'help_text', 'default_value', 'placeholder', 'required',
            'min_length', 'max_length', 'min_value', 'max_value', 'many'
        ]

        if getattr(field, 'read_only', False):
            return None

        for attr in attrs:
            value = getattr(field, attr, None)
            if value is not None and value != '':
                field_info[attr] = force_text(value, strings_only=True)

        if 'label' not in field_info:
            field_info['label'] = field_name.replace('_', ' ').title()

        if hasattr(field, 'view_name'):
            list_view = field.view_name.replace('-detail', '-list')
            base_url = reverse(list_view, request=self.request)
            field_info['type'] = 'select'
            field_info['url'] = base_url
            if hasattr(field, 'query_params'):
                field_info['url'] += '?%s' % urlencode(field.query_params)
            field_info['value_field'] = getattr(field, 'value_field', 'url')
            field_info['display_name_field'] = getattr(field, 'display_name_field', 'display_name')

        if hasattr(field, 'choices') and not hasattr(field, 'queryset'):
            field_info['choices'] = [
                {
                    'value': choice_value,
                    'display_name': force_text(choice_name, strings_only=True)
                }
                for choice_value, choice_name in field.choices.items()
            ]

        return field_info
<commit_msg>Skip tags field in action because it is needed only for resource creation [WAL-1223]<commit_after>from collections import OrderedDict

from django.utils.encoding import force_text
from django.utils.http import urlencode
from rest_framework import exceptions
from rest_framework.metadata import SimpleMetadata
from rest_framework.request import clone_request
from rest_framework.reverse import reverse

from nodeconductor.core.utils import sort_dict

class ActionSerializer(object):
    def __init__(self, func, name, request, view, resource):
        self.func = func
        self.name = name
        self.request = request
        self.resource = resource
        self.view = view

    def serialize(self):
        reason = self.get_reason()

        return {
            'title': self.get_title(),
            'method': self.get_method(),
            'destructive': self.is_destructive(),
            'url': self.get_url(),
            'reason': reason,
            'enabled': not reason
        }

    def is_destructive(self):
        if self.name == 'destroy':
            return True
        return getattr(self.func, 'destructive', False)

    def get_title(self):
        try:
            return getattr(self.func, 'title')
        except AttributeError:
            return self.name.replace('_', ' ').title()

    def get_reason(self):
        try:
            self.view.initial(self.request)
        except exceptions.APIException as e:
            return force_text(e)

    def get_method(self):
        if self.name == 'destroy':
            return 'DELETE'
        elif self.name == 'update':
            return 'PUT'
        return getattr(self.func, 'method', 'POST')

    def get_url(self):
        base_url = self.request.build_absolute_uri()
        method = self.get_method()
        if method in ('DELETE', 'PUT'):
            return base_url
        return base_url + self.name + '/'

class ActionsMetadata(SimpleMetadata):
    """
    Difference from SimpleMetadata class:
    1) Skip read-only fields, because options are used only for provisioning new resource.
    2) Don't expose choices for fields with queryset in order to reduce size of response.
    3) Attach actions metadata
    """
    def determine_metadata(self, request, view):
        self.request = request
        metadata = OrderedDict()
        if view.lookup_field in view.kwargs:
            metadata['actions'] = self.get_actions(request, view)
        else:
            metadata['actions'] = self.determine_actions(request, view)
        return metadata

    def get_actions(self, request, view):
        """
        Return metadata for resource-specific actions,
        such as start, stop, unlink
        """
        metadata = OrderedDict()
        actions = self.get_resource_actions(view)

        resource = view.get_object()
        for action_name, action in actions.items():
            if action_name == 'update':
                view.request = clone_request(request, 'PUT')
            else:
                view.action = action_name

            data = ActionSerializer(action, action_name, request, view, resource)
            metadata[action_name] = data.serialize()
            if not metadata[action_name]['enabled']:
                continue
            fields = self.get_action_fields(view, action_name, resource)
            if not fields:
                metadata[action_name]['type'] = 'button'
            else:
                metadata[action_name]['type'] = 'form'
                metadata[action_name]['fields'] = fields

            view.action = None
            view.request = request

        return metadata

    @classmethod
    def get_resource_actions(cls, view):
        actions = {}
        disabled_actions = getattr(view.__class__, 'disabled_actions', [])

        for key in dir(view.__class__):
            callback = getattr(view.__class__, key)
            if getattr(callback, 'deprecated', False):
                continue
            if 'post' not in getattr(callback, 'bind_to_methods', []):
                continue
            if key in disabled_actions:
                continue
            actions[key] = callback

        if 'DELETE' in view.allowed_methods and 'destroy' not in disabled_actions:
            actions['destroy'] = view.destroy

        if 'PUT' in view.allowed_methods and 'update' not in disabled_actions:
            actions['update'] = view.update

        return sort_dict(actions)

    def get_action_fields(self, view, action_name, resource):
        """
        Get fields exposed by action's serializer
        """
        serializer = view.get_serializer(resource)
        fields = OrderedDict()
        if not isinstance(serializer, view.serializer_class) or action_name == 'update':
            fields = self.get_fields(serializer.fields)
        return fields

    def get_serializer_info(self, serializer):
        """
        Given an instance of a serializer, return a dictionary of metadata
        about its fields.
        """
        if hasattr(serializer, 'child'):
            # If this is a `ListSerializer` then we want to examine the
            # underlying child serializer instance instead.
            serializer = serializer.child
        return self.get_fields(serializer.fields)

    def get_fields(self, serializer_fields):
        """
        Get fields metadata skipping empty fields
        """
        fields = OrderedDict()
        for field_name, field in serializer_fields.items():
            # Skip tags field in action because it is needed only for resource creation
            # See also: WAL-1223
            if field_name == 'tags':
                continue
            info = self.get_field_info(field, field_name)
            if info:
                fields[field_name] = info
        return fields

    def get_field_info(self, field, field_name):
        """
        Given an instance of a serializer field, return a dictionary
        of metadata about it.
        """
        field_info = OrderedDict()
        field_info['type'] = self.label_lookup[field]
        field_info['required'] = getattr(field, 'required', False)

        attrs = [
            'label', 'help_text', 'default_value', 'placeholder', 'required',
            'min_length', 'max_length', 'min_value', 'max_value', 'many'
        ]

        if getattr(field, 'read_only', False):
            return None

        for attr in attrs:
            value = getattr(field, attr, None)
            if value is not None and value != '':
                field_info[attr] = force_text(value, strings_only=True)

        if 'label' not in field_info:
            field_info['label'] = field_name.replace('_', ' ').title()

        if hasattr(field, 'view_name'):
            list_view = field.view_name.replace('-detail', '-list')
            base_url = reverse(list_view, request=self.request)
            field_info['type'] = 'select'
            field_info['url'] = base_url
            if hasattr(field, 'query_params'):
                field_info['url'] += '?%s' % urlencode(field.query_params)
            field_info['value_field'] = getattr(field, 'value_field', 'url')
            field_info['display_name_field'] = getattr(field, 'display_name_field', 'display_name')

        if hasattr(field, 'choices') and not hasattr(field, 'queryset'):
            field_info['choices'] = [
                {
                    'value': choice_value,
                    'display_name': force_text(choice_name, strings_only=True)
                }
                for choice_value, choice_name in field.choices.items()
            ]

        return field_info
<|endoftext|>


<commit_before>"""Provide the InboxableMixin class."""

from ....const import API_PATH

class InboxableMixin(object):
    """Interface for RedditBase classes that originate from the inbox."""

    def block(self):
        """Block the user who sent the item.

        :returns: The json response from the server.

        .. note:: Reddit does not permit blocking users unless you have an
                  :class:`.Comment` or :class:`.Message` from them.

        """
        return self._reddit.post(API_PATH['block'], data={'id': self.fullname})

    def mark_read(self):
        """Mark the item as read."""
        self._reddit.inbox.mark_read([self])

    def mark_unread(self):
        """Mark the item as unread."""
        self._reddit.inbox.mark_unread([self])
<commit_msg>Minor grammar fix<commit_after>"""Provide the InboxableMixin class."""

from ....const import API_PATH

class InboxableMixin(object):
    """Interface for RedditBase classes that originate from the inbox."""

    def block(self):
        """Block the user who sent the item.

        :returns: The json response from the server.

        .. note:: Reddit does not permit blocking users unless you have a
                  :class:`.Comment` or :class:`.Message` from them.

        """
        return self._reddit.post(API_PATH['block'], data={'id': self.fullname})

    def mark_read(self):
        """Mark the item as read."""
        self._reddit.inbox.mark_read([self])

    def mark_unread(self):
        """Mark the item as unread."""
        self._reddit.inbox.mark_unread([self])
<|endoftext|>