<commit_before>"""
Views for the crowdfund application
"""

from django.core.urlresolvers import reverse, NoReverseMatch
from django.contrib import messages
from django.contrib.auth.decorators import login_required, user_passes_test
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import redirect
from django.utils.decorators import method_decorator
from django.views.generic import ListView, DetailView, CreateView

import actstream
from datetime import date, timedelta
import logging
import stripe

from muckrock.crowdfund.forms import CrowdfundProjectForm, \
                                     CrowdfundRequestPaymentForm, \
                                     CrowdfundProjectPaymentForm
from muckrock.crowdfund.models import CrowdfundRequest, CrowdfundProject
from muckrock.project.models import Project
from muckrock.settings import STRIPE_SECRET_KEY, STRIPE_PUB_KEY

logger = logging.getLogger(__name__)
stripe.api_key = STRIPE_SECRET_KEY

class CrowdfundRequestListView(ListView):
    """Lists active request crowdfunds"""
    model = CrowdfundRequest
    template_name = 'crowdfund/request_list.html'

    def get_context_data(self, **kwargs):
        """Add title and other data to context"""
        context = super(CrowdfundRequestListView, self).get_context_data(**kwargs)
        context['title'] = 'Requests needing funding'
        return context

    def get_queryset(self):
        """Only list open crowdfunds on unembargoed requests"""
        queryset = super(CrowdfundRequestListView, self).get_queryset()
        queryset = queryset.exclude(closed=True).exclude(date_due__lt=date.today())
        user = self.request.user
        if not user.is_staff:
            if user.is_authenticated():
                queryset = queryset.filter(Q(foia__embargo=False)|Q(foia__user=user))
            else:
                queryset = queryset.filter(foia__embargo=False)
        return queryset

class CrowdfundProjectListView(ListView):
    """Lists active project crowdfunds"""
    model = CrowdfundProject
    template_name = 'crowdfund/project_list.html'

    def get_context_data(self, **kwargs):
        """Add title and other data to context"""
        context = super(CrowdfundProjectListView, self).get_context_data(**kwargs)
        context['title'] = 'Projects needing funding'
        return context

    def get_queryset(self):
        """Only list open crowdfunds on public projects"""
        queryset = super(CrowdfundProjectListView, self).get_queryset()
        queryset = queryset.exclude(closed=True).exclude(date_due__lt=date.today())
        user = self.request.user
        if not user.is_staff:
            if user.is_authenticated():
                queryset = queryset.filter(Q(project__private=False)|Q(project__contributors=user))
            else:
                queryset = queryset.filter(project__private=False)
        return queryset

class CrowdfundDetailView(DetailView):
    """
    Presents details about a crowdfunding campaign,
    as well as providing a private endpoint for contributions.
    """
    form = None

    def get_form(self):
        """Returns a form or None"""
        return self.form

    def get_context_data(self, **kwargs):
        """Adds Stripe public key to context"""
        context = super(CrowdfundDetailView, self).get_context_data(**kwargs)
        context['stripe_pk'] = STRIPE_PUB_KEY
        return context

    def get_redirect_url(self):
        """Returns a url to redirect to"""
        redirect_url = reverse('index')
        try:
            crowdfund_object = self.get_object().get_crowdfund_object()
            redirect_url = crowdfund_object.get_absolute_url()
        except (AttributeError, NoReverseMatch) as exception:
            logging.error(exception)
        return redirect_url

    def return_error(self, request):
        """If AJAX, return HTTP 400 ERROR. Else, add a message to the session."""
        if request.is_ajax():
            return HttpResponse(400)
        else:
            messages.error(
                request,
                ('There was an error making your contribution. '
                'Your card has not been charged.')
            )
            return redirect(self.get_redirect_url())

    def post(self, request, **kwargs):
        """
        First we validate the payment form, so we don't charge someone's card by accident.
        Next, we charge their card. Finally, use the validated payment form to create and
        return a CrowdfundRequestPayment object.
        """
        token = request.POST.get('token')

        try:
            payment_form = self.get_form()
            # pylint:disable=not-callable
            payment_form = payment_form(request.POST)
            # pylint:enable=not-callable
        except TypeError:
            logging.error(('The subclassed object does not have a form attribute '
                           'so no payments can be made.'))
            raise ValueError('%s does not have its form attribute set.' % self.__class__)
        if payment_form.is_valid() and token:
            cleaned_data = payment_form.cleaned_data
            crowdfund = cleaned_data['crowdfund']
            amount = cleaned_data['amount']
            show = cleaned_data['show']
            user = request.user if request.user.is_authenticated() else None
            stripe_exceptions = (
                stripe.InvalidRequestError,
                stripe.CardError,
                stripe.APIConnectionError,
                stripe.AuthenticationError
            )
            try:
                payment = crowdfund.make_payment(token, amount, show, user)
                actstream.action.send(
                    payment,
                    verb='contributed',
                    target=crowdfund
                )
            except stripe_exceptions as payment_error:
                logging.error(payment_error)
                self.return_error(request)
            # if AJAX, return HTTP 200 OK
            # else, add a message to the session
            if request.is_ajax():
                return HttpResponse(200)
            else:
                messages.success(request, 'Thank you for your contribution!')
                return redirect(self.get_redirect_url())
        return self.return_error(request)

class CrowdfundRequestDetail(CrowdfundDetailView):
    """Specificies a detail view for crowdfunding requests."""
    model = CrowdfundRequest
    form = CrowdfundRequestPaymentForm
    template_name = 'crowdfund/request_detail.html'

class CrowdfundProjectDetail(CrowdfundDetailView):
    """Specifies a detail view for crowdfunding projects."""
    model = CrowdfundProject
    form = CrowdfundProjectPaymentForm
    template_name = 'crowdfund/project_detail.html'

class CrowdfundProjectCreateView(CreateView):
    """A creation view for project crowdfunding"""
    model = CrowdfundProject
    form_class = CrowdfundProjectForm
    template_name = 'project/crowdfund.html'

    @method_decorator(login_required)
    @method_decorator(user_passes_test(lambda u: u.is_staff))
    def dispatch(self, *args, **kwargs):
        """At the moment, only staff are allowed to create a project crowdfund."""
        return super(CrowdfundProjectCreateView, self).dispatch(*args, **kwargs)

    def get_project(self):
        """Returns the project based on the URL keyword arguments"""
        return self.get_object(queryset=Project.objects.all())

    def get_initial(self):
        """Sets defaults in crowdfund project form"""
        project = self.get_project()
        initial_name = 'Crowdfund the ' + project.title
        initial_date = date.today() + timedelta(30)
        return {
            'name': initial_name,
            'date_due': initial_date,
            'project': project.id
        }

    def generate_action(self):
        """Generates an activity stream action"""
        crowdfund = self.get_object()
        actstream.action.send(
            self.request.user,
            verb='created',
            action_object=crowdfund
        )

    def get_success_url(self):
        self.generate_action()
        project = self.get_project()
        return project.get_absolute_url()
<commit_msg>adds project action to crowdfund creation to close #595<commit_after>"""
Views for the crowdfund application
"""

from django.core.urlresolvers import reverse, NoReverseMatch
from django.contrib import messages
from django.contrib.auth.decorators import login_required, user_passes_test
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import redirect
from django.utils.decorators import method_decorator
from django.views.generic import ListView, DetailView, CreateView

import actstream
from datetime import date, timedelta
import logging
import stripe

from muckrock.crowdfund.forms import CrowdfundProjectForm, \
                                     CrowdfundRequestPaymentForm, \
                                     CrowdfundProjectPaymentForm
from muckrock.crowdfund.models import CrowdfundRequest, CrowdfundProject
from muckrock.project.models import Project
from muckrock.settings import STRIPE_SECRET_KEY, STRIPE_PUB_KEY

logger = logging.getLogger(__name__)
stripe.api_key = STRIPE_SECRET_KEY

class CrowdfundRequestListView(ListView):
    """Lists active request crowdfunds"""
    model = CrowdfundRequest
    template_name = 'crowdfund/request_list.html'

    def get_context_data(self, **kwargs):
        """Add title and other data to context"""
        context = super(CrowdfundRequestListView, self).get_context_data(**kwargs)
        context['title'] = 'Requests needing funding'
        return context

    def get_queryset(self):
        """Only list open crowdfunds on unembargoed requests"""
        queryset = super(CrowdfundRequestListView, self).get_queryset()
        queryset = queryset.exclude(closed=True).exclude(date_due__lt=date.today())
        user = self.request.user
        if not user.is_staff:
            if user.is_authenticated():
                queryset = queryset.filter(Q(foia__embargo=False)|Q(foia__user=user))
            else:
                queryset = queryset.filter(foia__embargo=False)
        return queryset

class CrowdfundProjectListView(ListView):
    """Lists active project crowdfunds"""
    model = CrowdfundProject
    template_name = 'crowdfund/project_list.html'

    def get_context_data(self, **kwargs):
        """Add title and other data to context"""
        context = super(CrowdfundProjectListView, self).get_context_data(**kwargs)
        context['title'] = 'Projects needing funding'
        return context

    def get_queryset(self):
        """Only list open crowdfunds on public projects"""
        queryset = super(CrowdfundProjectListView, self).get_queryset()
        queryset = queryset.exclude(closed=True).exclude(date_due__lt=date.today())
        user = self.request.user
        if not user.is_staff:
            if user.is_authenticated():
                queryset = queryset.filter(Q(project__private=False)|Q(project__contributors=user))
            else:
                queryset = queryset.filter(project__private=False)
        return queryset

class CrowdfundDetailView(DetailView):
    """
    Presents details about a crowdfunding campaign,
    as well as providing a private endpoint for contributions.
    """
    form = None

    def get_form(self):
        """Returns a form or None"""
        return self.form

    def get_context_data(self, **kwargs):
        """Adds Stripe public key to context"""
        context = super(CrowdfundDetailView, self).get_context_data(**kwargs)
        context['stripe_pk'] = STRIPE_PUB_KEY
        return context

    def get_redirect_url(self):
        """Returns a url to redirect to"""
        redirect_url = reverse('index')
        try:
            crowdfund_object = self.get_object().get_crowdfund_object()
            redirect_url = crowdfund_object.get_absolute_url()
        except (AttributeError, NoReverseMatch) as exception:
            logging.error(exception)
        return redirect_url

    def return_error(self, request):
        """If AJAX, return HTTP 400 ERROR. Else, add a message to the session."""
        if request.is_ajax():
            return HttpResponse(400)
        else:
            messages.error(
                request,
                ('There was an error making your contribution. '
                'Your card has not been charged.')
            )
            return redirect(self.get_redirect_url())

    def post(self, request, **kwargs):
        """
        First we validate the payment form, so we don't charge someone's card by accident.
        Next, we charge their card. Finally, use the validated payment form to create and
        return a CrowdfundRequestPayment object.
        """
        token = request.POST.get('token')

        try:
            payment_form = self.get_form()
            # pylint:disable=not-callable
            payment_form = payment_form(request.POST)
            # pylint:enable=not-callable
        except TypeError:
            logging.error(('The subclassed object does not have a form attribute '
                           'so no payments can be made.'))
            raise ValueError('%s does not have its form attribute set.' % self.__class__)
        if payment_form.is_valid() and token:
            cleaned_data = payment_form.cleaned_data
            crowdfund = cleaned_data['crowdfund']
            amount = cleaned_data['amount']
            show = cleaned_data['show']
            user = request.user if request.user.is_authenticated() else None
            stripe_exceptions = (
                stripe.InvalidRequestError,
                stripe.CardError,
                stripe.APIConnectionError,
                stripe.AuthenticationError
            )
            try:
                payment = crowdfund.make_payment(token, amount, show, user)
                actstream.action.send(
                    payment,
                    verb='contributed',
                    target=crowdfund
                )
            except stripe_exceptions as payment_error:
                logging.error(payment_error)
                self.return_error(request)
            # if AJAX, return HTTP 200 OK
            # else, add a message to the session
            if request.is_ajax():
                return HttpResponse(200)
            else:
                messages.success(request, 'Thank you for your contribution!')
                return redirect(self.get_redirect_url())
        return self.return_error(request)

class CrowdfundRequestDetail(CrowdfundDetailView):
    """Specificies a detail view for crowdfunding requests."""
    model = CrowdfundRequest
    form = CrowdfundRequestPaymentForm
    template_name = 'crowdfund/request_detail.html'

class CrowdfundProjectDetail(CrowdfundDetailView):
    """Specifies a detail view for crowdfunding projects."""
    model = CrowdfundProject
    form = CrowdfundProjectPaymentForm
    template_name = 'crowdfund/project_detail.html'

class CrowdfundProjectCreateView(CreateView):
    """A creation view for project crowdfunding"""
    model = CrowdfundProject
    form_class = CrowdfundProjectForm
    template_name = 'project/crowdfund.html'

    @method_decorator(login_required)
    @method_decorator(user_passes_test(lambda u: u.is_staff))
    def dispatch(self, *args, **kwargs):
        """At the moment, only staff are allowed to create a project crowdfund."""
        return super(CrowdfundProjectCreateView, self).dispatch(*args, **kwargs)

    def get_project(self):
        """Returns the project based on the URL keyword arguments"""
        return self.get_object(queryset=Project.objects.all())

    def get_initial(self):
        """Sets defaults in crowdfund project form"""
        project = self.get_project()
        initial_name = 'Crowdfund the ' + project.title
        initial_date = date.today() + timedelta(30)
        return {
            'name': initial_name,
            'date_due': initial_date,
            'project': project.id
        }

    def get_success_url(self):
        """Generates actions before returning URL"""
        crowdfund = self.get_object()
        project = self.get_project()
        actstream.action.send(
            self.request.user,
            verb='created',
            action_object=crowdfund
        )
        actstream.action.send(
            self.request.user,
            varb='added',
            action_object=crowdfund,
            target=project
        )
        return project.get_absolute_url()
<|endoftext|>


<commit_before>from __future__ import print_function
from benchpress import util
import numpy as np

def model(N, B):
    """Construct pseudo-data representing price samples?"""

    S = B.random_array((N,))
    S = S*4.0-2.0 + 60.0 # Price is between 58-62

    return S

def CND(X):
    """
    Cumulative normal distribution.
    Helper function used by BS(...).
    """

    (a1,a2,a3,a4,a5) = (0.31938153, -0.356563782, 1.781477937, -1.821255978, 1.330274429)
    L = np.absolute(X)
    K = 1.0 / (1.0 + 0.2316419 * L)
    w = 1.0 - 1.0 / np.sqrt(2*np.pi)*np.exp(-L*L/2.) * \
        (a1 * K + \
         a2 * (K**2) + \
         a3 * (K**3) + \
         a4 * (K**4) + \
         a5 * (K**5))

    mask    = X<0
    w       = w * ~mask + (1.0-w)*mask

    return w

def BS(CallPutFlag, S, X, T, r, v):
    """Black Sholes Function."""

    d1 = (np.log(S/X)+(r+v*v/2.)*T)/(v*np.sqrt(T))
    d2 = d1-v*np.sqrt(T)
    if CallPutFlag=='c':
        return S*CND(d1)-X*np.exp(-r*T)*CND(d2)
    else:
        return X*np.exp(-r*T)*CND(-d2)-S*CND(-d1)

def price(S, I, flag='c', X=65.0, dT=(1.0/365.0), r=0.08, v=0.3, visualize=False):
    """Model price as a function of time."""

    T   = dT
    Ps  = []
    N   = len(S)
    for i in xrange (I):
        P = np.sum(BS(flag, S, X, T, r, v)) / N
        Ps.append(P)
        T += dT
        if visualize:   #NB: this is only for experiments
            np.visualize(P, "3d", 0, 0.0, 10)
        if util.Benchmark().bohrium:
            np.flush()

    return Ps

def main():
    B = util.Benchmark()
    (N, I) = B.size

    if B.inputfn:
        S = B.load_array()
    else:
        S = model(N, B)                     # Construct pseudo-data

    if B.dumpinput:
        B.dump_arrays("black_scholes", {'input': S})

    B.start()
    R = price(S, I, visualize=B.visualize)  # Run the model
    B.stop()
    B.pprint()

    for i in range(len(R)):                 # Convert to one array
        if hasattr(R[i], "copy2numpy"):
            R[i] = R[i].copy2numpy()[()]
    if B.outputfn:
        B.tofile(B.outputfn, {'res':R})

    if B.verbose:
        print (R)

if __name__ == "__main__":
    main()
<commit_msg>--dtype=float32 now works<commit_after>from __future__ import print_function
from benchpress import util
import numpy as np
import math

def model(N, B):
    """Construct pseudo-data representing price samples?"""

    if util.Benchmark().bohrium:
        S = numpy.random.random(N).astype(B.dtype)
    else:
        S = np.random.random(N).astype(B.dtype)
    S = np.array(S*4.0-2.0 + 60.0) # Price is between 58-62

    return S

def CND(X):
    """
    Cumulative normal distribution.
    Helper function used by BS(...).
    """

    (a1,a2,a3,a4,a5) = (0.31938153, -0.356563782, 1.781477937, -1.821255978, 1.330274429)
    L = np.absolute(X)
    K = 1.0 / (1.0 + 0.2316419 * L)
    w = 1.0 - 1.0 / math.sqrt(2*np.pi)*np.exp(-L*L/2.) * \
        (a1 * K + \
         a2 * (K**2) + \
         a3 * (K**3) + \
         a4 * (K**4) + \
         a5 * (K**5))

    mask    = X<0
    w       = w * ~mask + (1.0-w)*mask

    return w

def BS(CallPutFlag, S, X, T, r, v):
    """Black Sholes Function."""

    d1 = (np.log(S/X)+(r+v*v/2.)*T)/(v*math.sqrt(T))
    d2 = d1-v*math.sqrt(T)
    if CallPutFlag=='c':
        return S*CND(d1)-X*math.exp(-r*T)*CND(d2)
    else:
        return X*math.exp(-r*T)*CND(-d2)-S*CND(-d1)

def price(S, I, flag='c', X=65.0, dT=(1.0/365.0), r=0.08, v=0.3, visualize=False):
    """Model price as a function of time."""

    T   = dT
    Ps  = []
    N   = len(S)
    for i in xrange (I):
        P = np.sum(BS(flag, S, X, T, r, v)) / N
        Ps.append(P)
        T += dT
        if visualize:   #NB: this is only for experiments
            np.visualize(P, "3d", 0, 0.0, 10)
        if util.Benchmark().bohrium:
            np.flush()

    return Ps

def main():
    B = util.Benchmark()
    (N, I) = B.size

    if B.inputfn:
        S = B.load_array()
    else:
        S = model(N, B)                     # Construct pseudo-data

    if B.dumpinput:
        B.dump_arrays("black_scholes", {'input': S})

    B.start()
    R = price(S, I, visualize=B.visualize)  # Run the model
    B.stop()
    B.pprint()

    for i in range(len(R)):                 # Convert to one array
        if hasattr(R[i], "copy2numpy"):
            R[i] = R[i].copy2numpy()[()]
    if B.outputfn:
        B.tofile(B.outputfn, {'res':R})

    if B.verbose:
        print (R)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import pandas
import numpy as np
from currency_converter import CurrencyConverter
from datetime import datetime
from pytz import timezone, country_timezones
from sklearn import preprocessing
from os.path import join
import utils_data

dataset = pandas.read_csv(utils_data.get_path_input_raw())
print(dataset.head())
print("")

# throw away the columns we don't use
del dataset["Unnamed: 0"]   # this is just an arbitrary index
del dataset["id"]           # because I don't know what this is
del dataset["AccountID"]    # because I don't know what this is
del dataset["Email"]        # not relevant

# merge first and last name
dataset["Name"] = dataset["First Name"].map(str) + dataset["Last Name"]
del dataset["First Name"]
del dataset["Last Name"]

# rename the column names for consistency
dataset = dataset.rename(columns={'Merchant': 'MerchantID', 'Card': 'CardID', 'date': 'Date', 'target': 'Target',
                                  'GeoCode': 'PurchaseCountry'})

# convert Merchant, Card, FirstName, LastName, Email into integers
le = preprocessing.LabelEncoder()
dataset["MerchantID"] = le.fit_transform(dataset["MerchantID"])
dataset["CardID"] = le.fit_transform(dataset["CardID"])
dataset["Name"] = le.fit_transform(dataset["Name"])

# convert dates into datetime format
dataset["Date"] = dataset["Date"].apply(lambda date: datetime.strptime(date, '%Y-%m-%d %H:%M:%S'))

# where the GeoCode is unknown, use the card issuing Country (important for time conversion)
dataset.loc[dataset["PurchaseCountry"] == "--", "PurchaseCountry"] = dataset.loc[dataset["PurchaseCountry"] == "--", "Country"]
dataset.loc[dataset["PurchaseCountry"].isnull(), "PurchaseCountry"] = dataset.loc[dataset["PurchaseCountry"].isnull(), "Country"]

# add timezone info (Pacific Standard Time)
dataset["Date"] = dataset["Date"].apply(lambda date: date.replace(tzinfo=timezone('US/Pacific')))

# convert dates into local times (not that if it's ambiguous like Australia I just take the first entry
dataset["Date"] = dataset.apply(lambda d: d["Date"].astimezone(timezone(country_timezones(d["PurchaseCountry"])[0])), axis=1)

# remove the timezone info
dataset["Date"] = dataset.apply(lambda d: d["Date"].replace(tzinfo=None), axis=1)

# convert currencies into EUR (using the conversion rate from the date of purchase)
c = CurrencyConverter(fallback_on_missing_rate=True)
dataset["Amount"] = dataset.apply(lambda d: round(c.convert(d["Amount"], d["Currency"], 'EUR', d["Date"]), 2), axis=1)

# the data goes from 12.5.15 to 2.1.17
# only three fraud cases are recorded in 2016, so we reduce to 01.01.16 - 31.12.16
dataset = dataset[dataset['Date'].apply(lambda d: d.year) == 2016]

# compare number of unique credit cards to number of unique names
print("num cards (all): ", dataset["CardID"].unique().shape[0])
print("num names (all): ", dataset["Name"].unique().shape[0])
print("num cards (genuine): ", dataset.loc[dataset["Target"] == 0, "CardID"].unique().shape[0])
print("num names (genuine): ", dataset.loc[dataset["Target"] == 0, "Name"].unique().shape[0])
print("num cards (fraud): ", dataset.loc[dataset["Target"] == 1, "CardID"].unique().shape[0])
print("num names (fraud): ", dataset.loc[dataset["Target"] == 1, "Name"].unique().shape[0])
print("")
# --> high overlap, we remove the names

# since the overlap between unique cards and unique names is very high, we remove the names
del dataset["Name"]

# analyse how often the card issuing country and purchase country differ
print("(card != purchase country) - (all): ", 100*round(np.sum(dataset["Country"] != dataset["PurchaseCountry"])/dataset.shape[0], 4), '%')
print("(card != purchase country) - (genuine): ", 100*round(np.sum(dataset.loc[dataset["Target"] == 0, "Country"] != dataset.loc[dataset["Target"] == 0, "PurchaseCountry"])/sum(dataset["Target"] == 0), 4), '%')
print("(card != purchase country) - (fraud): ", 100*round(np.sum(dataset.loc[dataset["Target"] == 1, "Country"] != dataset.loc[dataset["Target"] == 1, "PurchaseCountry"])/sum(dataset["Target"] == 1), 4), '%')
print("")
# --> we will delete PurchaseCountry after converting time zones!

# we only keep the 'card country' as a feature because the card and purchase
# country are mostly overlapping, and the purchase country data is incomplete
del dataset["PurchaseCountry"]

print(dataset.head())

dataset.to_csv(utils_data.get_path_input_preprocessed(), index_label=False)
<commit_msg>rearranged columns<commit_after>import pandas
import numpy as np
from currency_converter import CurrencyConverter
from datetime import datetime
from pytz import timezone, country_timezones
from sklearn import preprocessing
from os.path import join
import utils_data

dataset = pandas.read_csv(utils_data.get_path_input_raw())
print(dataset.head())
print("")

# throw away the columns we don't use
del dataset["Unnamed: 0"]   # this is just an arbitrary index
del dataset["id"]           # because I don't know what this is
del dataset["AccountID"]    # because I don't know what this is
del dataset["Email"]        # not relevant

# merge first and last name
dataset["Name"] = dataset["First Name"].map(str) + dataset["Last Name"]
del dataset["First Name"]
del dataset["Last Name"]

# rename the column names for consistency
dataset = dataset.rename(columns={'Merchant': 'MerchantID', 'Card': 'CardID', 'date': 'Date', 'target': 'Target',
                                  'GeoCode': 'PurchaseCountry'})

# convert Merchant, Card, FirstName, LastName, Email into integers
le = preprocessing.LabelEncoder()
dataset["MerchantID"] = le.fit_transform(dataset["MerchantID"])
dataset["CardID"] = le.fit_transform(dataset["CardID"])
dataset["Name"] = le.fit_transform(dataset["Name"])

# convert dates into datetime format
dataset["Date"] = dataset["Date"].apply(lambda date: datetime.strptime(date, '%Y-%m-%d %H:%M:%S'))

# where the GeoCode is unknown, use the card issuing Country (important for time conversion)
dataset.loc[dataset["PurchaseCountry"] == "--", "PurchaseCountry"] = dataset.loc[dataset["PurchaseCountry"] == "--", "Country"]
dataset.loc[dataset["PurchaseCountry"].isnull(), "PurchaseCountry"] = dataset.loc[dataset["PurchaseCountry"].isnull(), "Country"]

# add timezone info (Pacific Standard Time)
dataset["Date"] = dataset["Date"].apply(lambda date: date.replace(tzinfo=timezone('US/Pacific')))

# convert dates into local times (not that if it's ambiguous like Australia I just take the first entry
dataset["Date"] = dataset.apply(lambda d: d["Date"].astimezone(timezone(country_timezones(d["PurchaseCountry"])[0])), axis=1)

# remove the timezone info
dataset["Date"] = dataset.apply(lambda d: d["Date"].replace(tzinfo=None), axis=1)

# convert currencies into EUR (using the conversion rate from the date of purchase)
c = CurrencyConverter(fallback_on_missing_rate=True)
dataset["Amount"] = dataset.apply(lambda d: round(c.convert(d["Amount"], d["Currency"], 'EUR', d["Date"]), 2), axis=1)

# the data goes from 12.5.15 to 2.1.17
# only three fraud cases are recorded in 2016, so we reduce to 01.01.16 - 31.12.16
dataset = dataset[dataset['Date'].apply(lambda d: d.year) == 2016]

# compare number of unique credit cards to number of unique names
print("num cards (all): ", dataset["CardID"].unique().shape[0])
print("num names (all): ", dataset["Name"].unique().shape[0])
print("num cards (genuine): ", dataset.loc[dataset["Target"] == 0, "CardID"].unique().shape[0])
print("num names (genuine): ", dataset.loc[dataset["Target"] == 0, "Name"].unique().shape[0])
print("num cards (fraud): ", dataset.loc[dataset["Target"] == 1, "CardID"].unique().shape[0])
print("num names (fraud): ", dataset.loc[dataset["Target"] == 1, "Name"].unique().shape[0])
print("")
# --> high overlap, we remove the names

# since the overlap between unique cards and unique names is very high, we remove the names
del dataset["Name"]

# analyse how often the card issuing country and purchase country differ
print("(card != purchase country) - (all): ", 100*round(np.sum(dataset["Country"] != dataset["PurchaseCountry"])/dataset.shape[0], 4), '%')
print("(card != purchase country) - (genuine): ", 100*round(np.sum(dataset.loc[dataset["Target"] == 0, "Country"] != dataset.loc[dataset["Target"] == 0, "PurchaseCountry"])/sum(dataset["Target"] == 0), 4), '%')
print("(card != purchase country) - (fraud): ", 100*round(np.sum(dataset.loc[dataset["Target"] == 1, "Country"] != dataset.loc[dataset["Target"] == 1, "PurchaseCountry"])/sum(dataset["Target"] == 1), 4), '%')
print("")
# --> we will delete PurchaseCountry after converting time zones!

# we only keep the 'card country' as a feature because the card and purchase
# country are mostly overlapping, and the purchase country data is incomplete
del dataset["PurchaseCountry"]

# bring columns into convenient order
dataset = dataset[['Date', 'CardID', 'MerchantID', 'Amount', 'Currency', 'Country', 'Target']]

print(dataset.head())

dataset.to_csv(utils_data.get_path_input_preprocessed(), index_label=False)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#   Copyright (C) 2006 - 2016 by
#      Tarek Taha, KURI  <tataha@tarektaha.com>
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the
#   Free Software Foundation, Inc.,
#   51 Franklin Steet, Fifth Floor, Boston, MA  02111-1307, USA.

import rospy
import thread
import threading
import time
import mavros

from math import *
from mavros.utils import *
from mavros import setpoint as SP
from tf.transformations import quaternion_from_euler
import smach
import smach_ros

class Starting(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['waitingforGPS','GPSFixed'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('Waiting for GPS Fix')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'waitingforGPS'
        else:
            return 'GPSFixed'

class Exploring(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['moving2pose','hovering'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('CON 1: Executing state Exploring')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'moving2pose'
        else:
            return 'hovering'

class DetectingObjects(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['detectingObjects','objectsDetected'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('CON 1: Executing state DetectingObjects')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'detectingObjects'
        else:
            return 'objectsDetected'

class AllocatingTasks(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['planning','succeeded'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('Executing state AllocatingTasks')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'planning'
        else:
            return 'succeeded'

class Navigating2Object(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['navigating','reached'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('Executing state Navigating2Object')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'navigating'
        else:
            return 'reached'

class PickingObject(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['descending','picked'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('Executing state PickingObject')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'descending'
        else:
            return 'picked'

class Navigating2DropZone(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['navigating','hovering'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('Executing state Navigating2DropZone')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'navigating'
        else:
            return 'hovering'

class DroppingObject(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['dropping','dropped'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('Executing state DroppingObject')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'dropping'
        else:
            return 'dropped'

class WaitingForTask(smach.State):
    def __init__(self):
        smach.State.__init__(self, outcomes=['waiting','ready'])
        self.counter = 0

    def execute(self, userdata):
        rospy.loginfo('Executing state DroppingObject')
        rospy.sleep(1.0)
        if self.counter < 3:
            self.counter += 1
            return 'waiting'
        else:
            return 'ready'

def main():
    rospy.init_node('kuri_mbzirc_challenge3_state_machine')

    explorationUAV_SM = smach.StateMachine(outcomes=['done'])
    with explorationUAV_SM:
        smach.StateMachine.add('Exploring', Exploring(),
                               transitions={'moving2pose':'Exploring',
                                            'hovering':'DetectingObjects'})
        smach.StateMachine.add('DetectingObjects', DetectingObjects(),
                                    transitions={'detectingObjects':'DetectingObjects',
                                                 'objectsDetected':'done'})

    task_allocator_sm = smach.StateMachine(outcomes=['tasks_ready'])
    with task_allocator_sm:
        smach.StateMachine.add('AllocatingTasks', AllocatingTasks(),
                               transitions={'planning':'AllocatingTasks',
                                'succeeded':'tasks_ready'})

    uav_worker = smach.StateMachine(outcomes=['finished'])
    with uav_worker:
        smach.StateMachine.add('WaitingForTask', WaitingForTask(),
                               transitions={'waiting':'WaitingForTask',
                                'ready':'Navigating2Object'})
        smach.StateMachine.add('Navigating2Object', Navigating2Object(),
                               transitions={'navigating':'Navigating2Object',
                                'reached':'PickingObject'})
        smach.StateMachine.add('PickingObject', PickingObject(),
                               transitions={'descending':'PickingObject',
                                'picked':'Navigating2DropZone'})
        smach.StateMachine.add('Navigating2DropZone', Navigating2DropZone(),
                               transitions={'navigating':'Navigating2DropZone',
                                'hovering':'DroppingObject'})
        smach.StateMachine.add('DroppingObject', DroppingObject(),
                               transitions={'dropping':'DroppingObject',
                               'dropped':'finished'})

        cc = smach.Concurrence(
                    outcomes=['JustDoIt'],
                    default_outcome='JustDoIt',
                    outcome_map = {'JustDoIt':{'MainState':'finished','ExplorationUAV_SM':'done'}})

        with cc:
            smach.Concurrence.add('MainState', uav_worker)
            smach.Concurrence.add('ExplorationUAV_SM', explorationUAV_SM)

    main_sm = smach.StateMachine(outcomes=['WeWin'])
    with main_sm:
        smach.StateMachine.add('Start', Starting(),
                           transitions={'waitingforGPS':'Start',
                                        'GPSFixed':'CC'})
        smach.StateMachine.add('ExplorationUAV_SM', explorationUAV_SM,
                                   transitions={'done':'WeWin'})
        smach.StateMachine.add('MainState', uav_worker,
                           transitions={'finished':'WeWin'})
        smach.StateMachine.add('CC', cc,
                               transitions={'JustDoIt':'WeWin'})

    sis = smach_ros.IntrospectionServer('kuri_mbzirc_challenge3_state_machine_viewer', main_sm, '/kuri_mbzirc_challenge3_state_machine_root')
    sis.start()
    outcome = main_sm.execute()
    rospy.spin()
    sis.stop()
if __name__ == '__main__':
    main()

<commit_msg>- updated ch3 state-machine<commit_after>#!/usr/bin/env python
#
#   Copyright (C) 2006 - 2016 by
#      Tarek Taha, KURI  <tataha@tarektaha.com>
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the
#   Free Software Foundation, Inc.,
#   51 Franklin Steet, Fifth Floor, Boston, MA  02111-1307, USA.

import rospy
import thread
import threading
import time
import mavros

from math import *
from mavros.utils import *
from mavros import setpoint as SP
from tf.transformations import quaternion_from_euler
import smach
import smach_ros

class Starting(smach.State):
    def __init__(self,sleep_t):
        smach.State.__init__(self, outcomes=['waitingforGPS','GPSFixed'],output_keys=['starting_out'])
        self.counter = 0
	self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Waiting for GPS Fix\n\n')
        rospy.sleep(self.sleep_t)
        if self.counter < 3:
            self.counter += 1
            return 'waitingforGPS'
        else:
	    userdata.starting_out = 22 #userdata example
            return 'GPSFixed'

class Exploring(smach.State):
    def __init__(self,sleep_t):
        smach.State.__init__(self, outcomes=['moving2pose','hovering'],input_keys=['exploring_in'],output_keys=['exploring_out'])
        self.counter = 0
	self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Executing state Exploring\n\n')
        rospy.sleep(self.sleep_t)
        if self.counter < 3:
            self.counter += 1
            return 'moving2pose'
        else:
	    userdata.exploring_out = userdata.exploring_in+1 #userdata example
            return 'hovering'

class DetectingObjects(smach.State):
    def __init__(self,sleep_t):
        smach.State.__init__(self, outcomes=['detectingObjects','objectsDetected'],input_keys=['detecting_objects_in'],output_keys=['detecting_objects_out'] )
        self.counter = 0
	self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Executing state DetectingObjects\n\n')
        rospy.sleep(self.sleep_t)
        if self.counter < 3:
            self.counter += 1
            return 'detectingObjects'
        else:
	    userdata.detecting_objects_out=userdata.detecting_objects_in+1 #userdata example
            return 'objectsDetected'

class AllocatingTasks(smach.State):
    def __init__(self,sleep_t):
        smach.State.__init__(self, outcomes=['planning','succeeded'], input_keys=['allocating_tasks_in'],output_keys=['allocating_tasks_out_uav1','allocating_tasks_out_uav2'])
        self.counter = 0
	self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Executing state AllocatingTasks\n\n')
        userdata.allocating_tasks_out_uav1 = userdata.allocating_tasks_in + 1 #userdata example
	userdata.allocating_tasks_out_uav2 = userdata.allocating_tasks_in + 1 #userdata example
	#print(">>>>>> %f <<<<<<<<",userdata.allocating_tasks_in)
        rospy.sleep(self.sleep_t)
        if self.counter < 4:
            self.counter += 1
            return 'planning'
        else:
            return 'succeeded'

class Navigating2Object(smach.State):
    def __init__(self,sleep_t):
        smach.State.__init__(self, outcomes=['navigating','reached'], input_keys=['navigating_2_object_in'])
        self.counter = 0
	self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Executing state Navigating2Object\n\n')
        rospy.sleep(self.sleep_t)
        if self.counter < 3:
            self.counter += 1
            return 'navigating'
        else:
            return 'reached'

class PickingObject(smach.State):
    def __init__(self,sleep_t):
        smach.State.__init__(self, outcomes=['descending','picked','object_fell'])
        self.counter = 0
	self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Executing state PickingObject\n\n')
        rospy.sleep(self.sleep_t)
        if self.counter < 3:
            self.counter += 1
            return 'descending'
        else:
            return 'picked'
	return 'object_fell'

class Navigating2DropZone(smach.State):
    def __init__(self,sleep_t):
        smach.State.__init__(self, outcomes=['navigating','hovering'], input_keys=['navigating_2dropzone_in'])
        self.counter = 0
	self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Executing state Navigating2DropZone\n\n')
        rospy.sleep(self.sleep_t)
        if self.counter < 3:
            self.counter += 1
            return 'navigating'
        else:
            return 'hovering'

class DroppingObject(smach.State):
    def __init__(self,sleep_t):
        smach.State.__init__(self, outcomes=['dropping','dropped','dropping_fault'])
        self.counter = 0
	self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Executing state DroppingObject\n\n')
        rospy.sleep(self.sleep_t)
        if self.counter < 3:
            self.counter += 1
            return 'dropping'
        else:
            return 'dropped'
	return 'dropping_fault'

class WaitingForTask(smach.State):
    def __init__(self, sleep_t):
        smach.State.__init__(self, outcomes=['waiting','ready'],input_keys=['waiting_for_tasks_in'],output_keys=['waiting_for_tasks_out'])
        self.counter = 0
        self.sleep_t=sleep_t
    def execute(self, userdata):
        rospy.loginfo('Executing state WaitingForTask\n\n')
        #userdata.waiting_for_tasks_out=1
        rospy.sleep(self.sleep_t)
        if self.counter < 3:
            self.counter += 1
            return 'waiting'
        else:
	    userdata.waiting_for_tasks_out = userdata.waiting_for_tasks_in + 1 #userdata example
            return 'ready'

def main():
    rospy.init_node('kuri_mbzirc_challenge3_state_machine')

    ### Explorer UAV
    uav_exlorer_sm = smach.StateMachine(outcomes=['done'],
					input_keys=['explorer_gps_in'],
					output_keys=['explorer_objects_out']
				       )
    with uav_exlorer_sm:
        smach.StateMachine.add('Exploring', Exploring(0.5),
                               transitions={
					    'moving2pose':'Exploring',
                                            'hovering':'DetectingObjects'
					   },
			       remapping={
					  'exploring_in':'explorer_gps_in',
					  'exploring_out':'waypoints'
					 }
			       )
        smach.StateMachine.add('DetectingObjects', DetectingObjects(0.5),
                               transitions={
					    'detectingObjects':'DetectingObjects',
                                            'objectsDetected':'done'
					   },
			       remapping={
					  'detecting_objects_in' : 'waypoints',
					  'detecting_objects_out':'explorer_objects_out'
					 }
			      )

    ### Task Allocator
    task_allocator_sm = smach.StateMachine(outcomes=['tasks_ready'],
					   input_keys=['task_allocator_in'],
					   output_keys=['task_allocator_out_uav1', 'task_allocator_out_uav2']
					   )
    with task_allocator_sm:
        smach.StateMachine.add('AllocatingTasks', AllocatingTasks(0.5),
                               transitions={
					    'planning':'AllocatingTasks',
					    'succeeded':'tasks_ready'
					   },
			       remapping={
					  'allocating_tasks_in':'task_allocator_in',
					  'allocating_tasks_out_uav1':'task_allocator_out_uav1',
					  'allocating_tasks_out_uav2':'task_allocator_out_uav2'
					 }
			       )

    ### UAV Worker 1
    uav_worker1_sm = smach.StateMachine(outcomes=['finished', 'task_failed'], input_keys=['uav1_worker_tasks_in'])
    with uav_worker1_sm:
        smach.StateMachine.add('WaitingForTask', WaitingForTask(0.5),
                               transitions={
					    'waiting':'WaitingForTask',
					    'ready':'Navigating2Object'
					   },
			       remapping={
					    'waiting_for_tasks_in':'uav1_worker_tasks_in',
					    'waiting_for_tasks_out':'object'
					 }
			       )
        smach.StateMachine.add('Navigating2Object', Navigating2Object(0.5),
                               transitions={
					    'navigating':'Navigating2Object',
					    'reached':'PickingObject'},
			       remapping={
					    'navigating_2_object_in':'object'
					 }
			       )
        smach.StateMachine.add('PickingObject', PickingObject(0.5),
                               transitions={
					    'descending':'PickingObject',
					    'picked':'Navigating2DropZone',
					    'object_fell':'task_failed'

					      }

			       )
        smach.StateMachine.add('Navigating2DropZone', Navigating2DropZone(0.5),
                               transitions={
					    'navigating':'Navigating2DropZone',
					    'hovering':'DroppingObject'
					   }
			       )
        smach.StateMachine.add('DroppingObject', DroppingObject(0.5),
                               transitions={
					    'dropping':'DroppingObject',
					    'dropped':'finished',
					    'dropping_fault':'task_failed'

					   }
			       )

    ### UAV Worker 2
    uav_worker2_sm = smach.StateMachine(outcomes=['finished', 'task_failed'], input_keys=['uav2_worker_tasks_in'])
    with uav_worker2_sm:
        smach.StateMachine.add('WaitingForTask', WaitingForTask(1),
                               transitions={
					    'waiting':'WaitingForTask',
					    'ready':'Navigating2Object'
					   },
			       remapping={
					    'waiting_for_tasks_in':'uav2_worker_tasks_in',
					    'waiting_for_tasks_out':'object'
					 }
			       )
        smach.StateMachine.add('Navigating2Object', Navigating2Object(1),
                               transitions={
					    'navigating':'Navigating2Object',
					    'reached':'PickingObject'},
			       remapping={
					    'navigating_2_object_in':'object'
					 }
			       )
        smach.StateMachine.add('PickingObject', PickingObject(1),
                               transitions={
					    'descending':'PickingObject',
					    'picked':'Navigating2DropZone',
					    'object_fell':'task_failed'

					      }

			       )
        smach.StateMachine.add('Navigating2DropZone', Navigating2DropZone(1),
                               transitions={
					    'navigating':'Navigating2DropZone',
					    'hovering':'DroppingObject'
					   }
			       )
        smach.StateMachine.add('DroppingObject', DroppingObject(1),
                               transitions={
					    'dropping':'DroppingObject',
					    'dropped':'finished',
					    'dropping_fault':'task_failed'

					   }
			       )

    ### concurruncy
    cc = smach.Concurrence(
                    outcomes=['succeeded','failed'],
                    default_outcome='succeeded',
                    outcome_map = {
				    'succeeded':{'UAV_Worker1':'finished','UAV_Worker2':'finished'},
				    'failed':{'UAV_Worker1':'task_failed','UAV_Worker2':'task_failed'},
				    'failed':{'UAV_Worker1':'task_failed','UAV_Worker2':'finished'},
				    'failed':{'UAV_Worker1':'finished','UAV_Worker2':'task_failed'}
				  },
                    input_keys=['cc_uav1_tasks_in','cc_uav2_tasks_in']
                    #input_keys=['cc_in']
                    )

    with cc:
	    ## the task allocator inside concurruncy have problem with userdata since the tasks are needed by the states of the uav_workers
	    #smach.Concurrence.add('Task_Allocator', task_allocator_sm,
				    #remapping={
						#'task_allocator_in':'cc_in',
						#'task_allocator_out_uav1':'UAV1_tasks',
						#'task_allocator_out_uav2':'UAV2_tasks'
					      #}
				  #)
            smach.Concurrence.add('UAV_Worker1', uav_worker1_sm,
				    remapping={
						'uav1_worker_tasks_in':'cc_uav1_tasks_in'
						#'uav1_worker_tasks_in':'UAV1_tasks'
					      }

				  )
            smach.Concurrence.add('UAV_Worker2', uav_worker2_sm,
				    remapping={
						'uav2_worker_tasks_in':'cc_uav2_tasks_in'
						#'uav2_worker_tasks_in':'UAV2_tasks'
					      }

				  )

    ### MAIN
    main_sm = smach.StateMachine(outcomes=['Mission_Accomplished'])
    with main_sm:
        smach.StateMachine.add('Start', Starting(1),
				    transitions={'waitingforGPS':'Start',
                                        'GPSFixed':'UAV_Explorer'},
				    remapping={'starting_out':'uav_gps_loc'}
			      )
        smach.StateMachine.add('UAV_Explorer', uav_exlorer_sm,
				    transitions={'done':'Task_Allocator'}, #change CC to task_allocator if you want to remove taskallocator from the concurruncy
				    remapping={
						'explorer_gps_in':'uav_gps_loc',
						'explorer_objects_out':'objects_locations'
					      }
			      )

	#if we don't want to put the task_allocator in the concurruncy
	smach.StateMachine.add('Task_Allocator', task_allocator_sm,
				    transitions={'tasks_ready':'CC'},
				    remapping={
						'task_allocator_in':'objects_locations',
						'task_allocator_out_uav1':'UAV1_tasks',
  						'task_allocator_out_uav2':'UAV2_tasks'
					      }
			      )

        smach.StateMachine.add('CC', cc,
                               transitions={'succeeded':'Mission_Accomplished',
					    'failed':'UAV_Explorer'
					   },
                               remapping={
					    'cc_in':'objects_locations',
   					    'cc_uav1_tasks_in':'UAV1_tasks',
   					    'cc_uav2_tasks_in':'UAV2_tasks'
					 }
                              )

    sis = smach_ros.IntrospectionServer('kuri_mbzirc_challenge3_state_machine_viewer', main_sm, '/kuri_mbzirc_challenge3_state_machine_root')
    sis.start()
    outcome = main_sm.execute()
    rospy.spin()
    sis.stop()
if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import asyncio
import os
import pytest
import sys
import tempfile
import time

import ray

from ray._private.test_utils import (Semaphore)

def test_nested_tasks(shutdown_only):
    ray.init(num_cpus=1)

    @ray.remote
    class Counter:
        def __init__(self):
            self.count = 0

        def inc(self):
            self.count += 1
            # Since we relex the cap after a timeout we can have slightly more
            # than 1 task. We should never have 20 though since that takes 2^20
            # * 10ms time.
            assert self.count < 20

        def dec(self):
            self.count -= 1

    counter = Counter.remote()

    @ray.remote(num_cpus=1)
    def g():
        return None

    @ray.remote(num_cpus=1)
    def f():
        ray.get(counter.inc.remote())
        res = ray.get(g.remote())
        ray.get(counter.dec.remote())
        return res

    ready, _ = ray.wait(
        [f.remote() for _ in range(1000)], timeout=60.0, num_returns=1000)
    assert len(ready) == 1000, len(ready)
    # Ensure the assertion in `inc` didn't fail.
    ray.get(ready)

def test_recursion(shutdown_only):
    ray.init(num_cpus=1)

    @ray.remote
    def summer(n):
        if n == 0:
            return 0
        return n + ray.get(summer.remote(n - 1))

    assert ray.get(summer.remote(10)) == sum(range(11))

def test_out_of_order_scheduling(shutdown_only):
    """Ensure that when a task runs before its dependency, and they're of the same
       scheduling class, the dependency is eventually able to run."""
    ray.init(num_cpus=1)

    @ray.remote
    def foo(arg, path):
        ref, = arg
        should_die = not os.path.exists(path)
        with open(path, "w") as f:
            f.write("")
        if should_die:
            print("dying!!!")
            os._exit(-1)

        if ref:
            print("hogging the only available slot for a while")
            ray.get(ref)
            return "done!"

    with tempfile.TemporaryDirectory() as tmpdir:
        path = f"{tmpdir}/temp.txt"
        first = foo.remote((None, ), path)
        second = foo.remote((first, ), path)
        print(ray.get(second))

def test_limit_concurrency(shutdown_only):
    ray.init(num_cpus=1)

    block_task = Semaphore.remote(0)
    block_driver = Semaphore.remote(0)

    ray.get([block_task.locked.remote(), block_driver.locked.remote()])

    @ray.remote(num_cpus=1)
    def foo():
        ray.get(block_driver.release.remote())
        ray.get(block_task.acquire.remote())

    refs = [foo.remote() for _ in range(20)]

    block_driver_refs = [block_driver.acquire.remote() for _ in range(20)]

    # Some of the tasks will run since we relax the cap, but not all because it
    # should take exponentially long for the cap to be increased.
    ready, not_ready = ray.wait(block_driver_refs, timeout=10, num_returns=20)
    assert len(not_ready) >= 1

    # Now the first instance of foo finishes, so the second starts to run.
    ray.get([block_task.release.remote() for _ in range(19)])

    ready, not_ready = ray.wait(block_driver_refs, timeout=10, num_returns=20)
    assert len(not_ready) == 0

    ready, not_ready = ray.wait(refs, num_returns=20, timeout=15)
    assert len(ready) == 19
    assert len(not_ready) == 1

@pytest.mark.skipif(sys.platform == "win32", reason="Times out on windows.")
def test_zero_cpu_scheduling(shutdown_only):
    ray.init(num_cpus=1)

    block_task = Semaphore.remote(0)
    block_driver = Semaphore.remote(0)

    @ray.remote(num_cpus=0)
    def foo():
        ray.get(block_driver.release.remote())
        ray.get(block_task.acquire.remote())

    foo.remote()
    foo.remote()

    ray.get(block_driver.acquire.remote())

    block_driver_ref = block_driver.acquire.remote()

    # Both tasks should be running, so the driver should be unblocked.
    ready, not_ready = ray.wait([block_driver_ref], timeout=1)
    assert len(not_ready) == 0

def test_exponential_wait(shutdown_only):
    ray.init(num_cpus=2)

    num_tasks = 4

    @ray.remote(num_cpus=0)
    class Barrier:
        def __init__(self, limit):
            self.i = 0
            self.limit = limit

        async def join(self):
            self.i += 1
            while self.i < self.limit:
                await asyncio.sleep(1)

    b = Barrier.remote(num_tasks)

    @ray.remote
    def f(i, start):
        delta = time.time() - start
        print("Launch", i, time.time() - start)
        ray.get(b.join.remote())
        return delta

    start = time.time()
    results = ray.get([f.remote(i, start) for i in range(num_tasks)])

    last_wait = results[-1] - results[-2]
    second_last = results[-2] - results[-3]

    # Assert that last_wwait / second_last ~= 2, with a healthy buffer since ci
    # is noisy.
    assert 1 < last_wait / second_last < 4
    assert 2 < last_wait < 4

if __name__ == "__main__":
    import pytest
    os.environ["RAY_worker_cap_enabled"] = "true"
    sys.exit(pytest.main(["-v", __file__]))
<commit_msg>[windows][ci] Increase in `num_tasks` for stable wait times in `test_worker_capping:test_exponential_wait` (#21051)<commit_after>import asyncio
import os
import pytest
import sys
import tempfile
import time

import ray

from ray._private.test_utils import (Semaphore)

def test_nested_tasks(shutdown_only):
    ray.init(num_cpus=1)

    @ray.remote
    class Counter:
        def __init__(self):
            self.count = 0

        def inc(self):
            self.count += 1
            # Since we relex the cap after a timeout we can have slightly more
            # than 1 task. We should never have 20 though since that takes 2^20
            # * 10ms time.
            assert self.count < 20

        def dec(self):
            self.count -= 1

    counter = Counter.remote()

    @ray.remote(num_cpus=1)
    def g():
        return None

    @ray.remote(num_cpus=1)
    def f():
        ray.get(counter.inc.remote())
        res = ray.get(g.remote())
        ray.get(counter.dec.remote())
        return res

    ready, _ = ray.wait(
        [f.remote() for _ in range(1000)], timeout=60.0, num_returns=1000)
    assert len(ready) == 1000, len(ready)
    # Ensure the assertion in `inc` didn't fail.
    ray.get(ready)

def test_recursion(shutdown_only):
    ray.init(num_cpus=1)

    @ray.remote
    def summer(n):
        if n == 0:
            return 0
        return n + ray.get(summer.remote(n - 1))

    assert ray.get(summer.remote(10)) == sum(range(11))

def test_out_of_order_scheduling(shutdown_only):
    """Ensure that when a task runs before its dependency, and they're of the same
       scheduling class, the dependency is eventually able to run."""
    ray.init(num_cpus=1)

    @ray.remote
    def foo(arg, path):
        ref, = arg
        should_die = not os.path.exists(path)
        with open(path, "w") as f:
            f.write("")
        if should_die:
            print("dying!!!")
            os._exit(-1)

        if ref:
            print("hogging the only available slot for a while")
            ray.get(ref)
            return "done!"

    with tempfile.TemporaryDirectory() as tmpdir:
        path = f"{tmpdir}/temp.txt"
        first = foo.remote((None, ), path)
        second = foo.remote((first, ), path)
        print(ray.get(second))

def test_limit_concurrency(shutdown_only):
    ray.init(num_cpus=1)

    block_task = Semaphore.remote(0)
    block_driver = Semaphore.remote(0)

    ray.get([block_task.locked.remote(), block_driver.locked.remote()])

    @ray.remote(num_cpus=1)
    def foo():
        ray.get(block_driver.release.remote())
        ray.get(block_task.acquire.remote())

    refs = [foo.remote() for _ in range(20)]

    block_driver_refs = [block_driver.acquire.remote() for _ in range(20)]

    # Some of the tasks will run since we relax the cap, but not all because it
    # should take exponentially long for the cap to be increased.
    ready, not_ready = ray.wait(block_driver_refs, timeout=10, num_returns=20)
    assert len(not_ready) >= 1

    # Now the first instance of foo finishes, so the second starts to run.
    ray.get([block_task.release.remote() for _ in range(19)])

    ready, not_ready = ray.wait(block_driver_refs, timeout=10, num_returns=20)
    assert len(not_ready) == 0

    ready, not_ready = ray.wait(refs, num_returns=20, timeout=15)
    assert len(ready) == 19
    assert len(not_ready) == 1

@pytest.mark.skipif(sys.platform == "win32", reason="Times out on windows.")
def test_zero_cpu_scheduling(shutdown_only):
    ray.init(num_cpus=1)

    block_task = Semaphore.remote(0)
    block_driver = Semaphore.remote(0)

    @ray.remote(num_cpus=0)
    def foo():
        ray.get(block_driver.release.remote())
        ray.get(block_task.acquire.remote())

    foo.remote()
    foo.remote()

    ray.get(block_driver.acquire.remote())

    block_driver_ref = block_driver.acquire.remote()

    # Both tasks should be running, so the driver should be unblocked.
    ready, not_ready = ray.wait([block_driver_ref], timeout=1)
    assert len(not_ready) == 0

def test_exponential_wait(shutdown_only):
    ray.init(num_cpus=2)

    num_tasks = 6

    @ray.remote(num_cpus=0)
    class Barrier:
        def __init__(self, limit):
            self.i = 0
            self.limit = limit

        async def join(self):
            self.i += 1
            while self.i < self.limit:
                await asyncio.sleep(1)

    b = Barrier.remote(num_tasks)

    @ray.remote
    def f(i, start):
        delta = time.time() - start
        print("Launch", i, time.time() - start)
        ray.get(b.join.remote())
        return delta

    start = time.time()
    results = ray.get([f.remote(i, start) for i in range(num_tasks)])

    last_wait = results[-1] - results[-2]
    second_last = results[-2] - results[-3]

    # Assert that last_wwait / second_last ~= 2, with a healthy buffer since ci
    # is noisy.
    assert second_last < last_wait < 4 * second_last
    assert 7 < last_wait

if __name__ == "__main__":
    import pytest
    os.environ["RAY_worker_cap_enabled"] = "true"
    sys.exit(pytest.main(["-v", __file__]))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import os.path
from os.path import join as pj
import glob
import platform

from jupyter_client.kernelspec import KernelSpecManager, KernelSpec, NoSuchKernel
from ipykernel.kernelspec import RESOURCES

from traitlets import List

__all__ = ['EnvironmentKernelSpecManager']

try:
    import conda.config
    HAVE_CONDA = True

except ImportError:
    HAVE_CONDA = False

try:
    FileNotFoundError
except NameError:
    #py2
    FileNotFoundError = IOError

class EnvironmentKernelSpecManager(KernelSpecManager):
    """
    A Jupyter Kernel manager which dyamically checks for Environments

    Given a list of base directories, this class searches for the pattern::

        BASE_DIR/NAME/{bin|Skript}/ipython

    where NAME is taken to be the name of the environment.
    """

    # Take the default home DIR for conda and virtualenv as the default
    _default_conda_dirs = ['~/.conda/envs/']
    _default_virtualenv_dirs = ['~/.virtualenvs']

    # Check for the CONDA_ENV_PATH variable and add it to the list if set.
    if os.environ.get('CONDA_ENV_PATH', False):
        _default_conda_dirs.append(os.environ['CONDA_ENV_PATH'].split('envs')[0])

    # If we are running inside the root conda env can get all the env dirs:
    if HAVE_CONDA:
        _default_conda_dirs += conda.config.envs_dirs

    # Remove any duplicates
    _default_conda_dirs = list(set(map(os.path.expanduser, _default_conda_dirs)))

    conda_env_dirs = List(
        _default_conda_dirs, config=True,
        help="List of directories in which are conda environments."
    )

    virtualenv_env_dirs = List(
        _default_virtualenv_dirs, config=True,
        help="List of directories in which are virtualenv environments."
    )

    blacklist_envs = List(
        ["conda__build"], config=True,
        help="Environments which should not be used even if a ipykernel exists in it."
    )

    whitelist_envs = List(
        [], config=True,
        help="Environments which should be used (overwrites a blacklist)."
    )

    def __init__(self, *args, **kwargs):
        super(EnvironmentKernelSpecManager, self).__init__(*args, **kwargs)
        self.log.info("Using EnvironmentKernelSpecManager...")

    def validate_env(self, envname):
        """
        Check the name of the environment against the black list and the
        whitelist. If a whitelist is specified only it is checked.
        """
        if self.whitelist_envs and envname in self.whitelist_envs:
            return True
        elif self.whitelist_envs:
            return False

        if self.blacklist_envs and envname not in self.blacklist_envs:
            return True
        elif self.blacklist_envs:
            # If there is just a True, all envs are blacklisted
            return False
        else:
            return True

    def _find_conda_env_paths_from_conda(self):
        """Returns a list of path as given by `conda env list --json`.

        Returns empty list, if conda couldn't be called.
        """
        # use the cache, because using conda all the time is slow...
        import subprocess
        import json
        try:
            p = subprocess.Popen(
                ['conda', 'env', 'list', '--json'],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE)
            comm = p.communicate()
            output = comm[0].decode()
            if p.returncode != 0 or len(output) == 0:
                self.log.error("Couldn't call 'conda' to get the environments. "
                               "Output:\n%s" , str(comm))
                return []
        except FileNotFoundError:
            self.log.error("'conda' not found in path.")
            return []
        output = json.loads(output)
        envs = output["envs"]
        #self.log.info("Found the following kernels from conda: %s", ", ".join(envs))
        return envs

    def _find_conda_env_path_from_config(self):
        """Returns a list of env paths as by using the configured conda env basedir"""
        return self._find_env_paths_in_basedirs(self.conda_env_dirs)

    def _find_virtualenv_env_path_from_config(self):
        """Returns a list of env paths as by using the configured virtualenv basedir"""
        return self._find_env_paths_in_basedirs(self.virtualenv_env_dirs)

    def _validate_ipython_path(self, env_path):
        """Returns whether a ipython executable is in that env"""
        # We validate a dir if it has a ipython executable in one of the dirs
        # which ends up in PATH. The proper way would probably use the ipykernel
        # as ipython itself is no garantee that the env can also be used as a kernel
        # but it is close...
        ipython = "ipython.exe" if platform.system() == "Windows" else "ipython"
        for search_path in ['bin', 'Scripts']:
            ipython_path = pj(env_path, search_path, ipython)
            if os.path.exists(ipython_path):
                return True
        return False

    def _find_env_paths_in_basedirs(self, base_dirs):
        """Returns all potential envs in a basedir"""
        # get potential env path in the base_dirs
        envs = []
        for base_dir in base_dirs:
            envs.extend(glob.glob(pj(os.path.expanduser(base_dir), '*', '')))
        #self.log.info("Found the following kernels from config: %s", ", ".join(venvs))

        # filter out envs which do not contain a ipython (~= ipkernel)
        envs = [env for env in envs if self._validate_ipython_path(env)]

        return envs

    def _convert_to_envs(self, env_paths, nametemplate):
        """Returns a dict of kernel_name -> path

        kernelname is build by using the template.
        """
        envs = {}
        for venv_dir in env_paths:
            venv_name = os.path.split(venv_dir)[1]
            kernel_name = nametemplate.format(venv_name)
            kernel_name = kernel_name.lower()
            if kernel_name in envs:
                self.log.error("Duplicate env kernels: %s would both point to %s and %s. Using the first!", kernel_name, envs[kernel_name], venv_dir)
                continue
            if self.validate_env(kernel_name):
                envs.update({kernel_name: venv_dir})
            else:
                self.log.info("Not considered env kernel (blacklisted): %s", kernel_name)
        return envs

    def _find_conda_envs(self):
        """Returns conda envs as dict of name -> path"""
        paths = self._find_conda_env_path_from_config()
        paths.extend(self._find_conda_env_paths_from_conda())
        paths = list(set(paths))

        # can't use '/' as that results in javascript errors :-/
        # need to use something which matches \w to get logos:
        # https://github.com/jupyter/notebook/issues/853
        templ = "conda_{}"
        return self._convert_to_envs(paths, templ)

    def _find_virtualenv_envs(self):
        """Returns virtualenv envs as dict of name -> path"""
        paths = self._find_virtualenv_env_path_from_config()

        templ = "virtualenv_{}"
        return self._convert_to_envs(paths, templ)

    def find_envs(self):
        """Returns for all envs as dict of name -> path"""

        # This is called much too often and the conda calls are really expensive :-(
        if hasattr(self, "_find_envs_cache"):
            return getattr(self, "_find_envs_cache")

        envs = {}
        envs.update(self._find_conda_envs())
        envs.update(self._find_virtualenv_envs())
        if envs:
            self.log.info("Found the following kernels for environments: %s", ", ".join(list(envs)))
        else:
            self.log.info("Found no kernels from environments!")
        self._find_envs_cache = envs
        return envs

    def _build_kernel_specs(self):
        """Returns the dict of name -> kernel_spec for all environments"""

        # This is called much too often and the conda calls are really expensive :-(
        if hasattr(self, "_build_kernel_specs_cache"):
            return getattr(self, "_build_kernel_specs_cache")

        venv_dirs = self.find_envs()
        kspecs = {}
        if platform.system() == "Windows":
            python_exe_name = "python.exe"
        else:
            python_exe_name = "python"

        for venv_name, venv_dir in venv_dirs.items():
            # conda on windows has python.exe directly in the env
            exe_name = pj(venv_dir, python_exe_name)
            if not os.path.exists(exe_name):
                exe_name = pj(venv_dir, "bin", python_exe_name)
            kspec_dict =  {"argv": [exe_name,
                                    "-m",
                                    "IPython.kernel",
                                    "-f",
                                    "{connection_file}"],
                           "language": "python",
                           "display_name": "Environment ({})".format(venv_name),
                           "env": {}}
            # This should probably use self.kernel_spec_class instead of the direct class
            kspecs.update({venv_name: KernelSpec(resource_dir=RESOURCES, **kspec_dict)})
        self._build_kernel_specs_cache = kspecs
        return kspecs

    def find_kernel_specs_for_envs(self):
        """Returns a dict mapping kernel names to resource directories."""
        # as the envs do not have logos (=resources) included, add in the
        # normal ones for python

        envs = self.find_envs()
        ret = {}
        for name in envs:
            ret[name] = RESOURCES
        return ret

    def find_kernel_specs(self):
        """Returns a dict mapping kernel names to resource directories."""
        # let real installed kernels overwrite envs with the same name:
        # this is the same order as the get_kernel_spec way, which also prefers
        # kernels from the jupyter dir over env kernels.
        specs = self.find_kernel_specs_for_envs()
        specs.update(super(EnvironmentKernelSpecManager, self).find_kernel_specs())
        return specs

    def get_kernel_spec(self, kernel_name):
        """Returns a :class:`KernelSpec` instance for the given kernel_name.

        Raises :exc:`NoSuchKernel` if the given kernel name is not found.
        """
        try:
            return super(EnvironmentKernelSpecManager, self).get_kernel_spec(kernel_name)
        except (NoSuchKernel, FileNotFoundError):
            venv_kernel_name = kernel_name.lower()
            specs = self._build_kernel_specs()
            if venv_kernel_name in specs:
                return specs[venv_kernel_name]
            else:
                raise NoSuchKernel(kernel_name)
<commit_msg>Make the display name configureable to enable better sorting<commit_after># -*- coding: utf-8 -*-

import os
import os.path
from os.path import join as pj
import glob
import platform

from jupyter_client.kernelspec import KernelSpecManager, KernelSpec, NoSuchKernel
from ipykernel.kernelspec import RESOURCES

from traitlets import List, Unicode

__all__ = ['EnvironmentKernelSpecManager']

try:
    import conda.config
    HAVE_CONDA = True

except ImportError:
    HAVE_CONDA = False

try:
    FileNotFoundError
except NameError:
    #py2
    FileNotFoundError = IOError

class EnvironmentKernelSpecManager(KernelSpecManager):
    """
    A Jupyter Kernel manager which dyamically checks for Environments

    Given a list of base directories, this class searches for the pattern::

        BASE_DIR/NAME/{bin|Skript}/ipython

    where NAME is taken to be the name of the environment.
    """

    # Take the default home DIR for conda and virtualenv as the default
    _default_conda_dirs = ['~/.conda/envs/']
    _default_virtualenv_dirs = ['~/.virtualenvs']

    # Check for the CONDA_ENV_PATH variable and add it to the list if set.
    if os.environ.get('CONDA_ENV_PATH', False):
        _default_conda_dirs.append(os.environ['CONDA_ENV_PATH'].split('envs')[0])

    # If we are running inside the root conda env can get all the env dirs:
    if HAVE_CONDA:
        _default_conda_dirs += conda.config.envs_dirs

    # Remove any duplicates
    _default_conda_dirs = list(set(map(os.path.expanduser, _default_conda_dirs)))

    conda_env_dirs = List(
        _default_conda_dirs, config=True,
        help="List of directories in which are conda environments."
    )

    virtualenv_env_dirs = List(
        _default_virtualenv_dirs, config=True,
        help="List of directories in which are virtualenv environments."
    )

    blacklist_envs = List(
        ["conda__build"], config=True,
        help="Environments which should not be used even if a ipykernel exists in it."
    )

    whitelist_envs = List(
        [], config=True,
        help="Environments which should be used (overwrites a blacklist)."
    )

    display_name_template = Unicode(
        u"Environment ({})", config=True,
        help="Template for the kernel name in the UI. Needs to include {} for the name."
    )

    def __init__(self, *args, **kwargs):
        super(EnvironmentKernelSpecManager, self).__init__(*args, **kwargs)
        self.log.info("Using EnvironmentKernelSpecManager...")

    def validate_env(self, envname):
        """
        Check the name of the environment against the black list and the
        whitelist. If a whitelist is specified only it is checked.
        """
        if self.whitelist_envs and envname in self.whitelist_envs:
            return True
        elif self.whitelist_envs:
            return False

        if self.blacklist_envs and envname not in self.blacklist_envs:
            return True
        elif self.blacklist_envs:
            # If there is just a True, all envs are blacklisted
            return False
        else:
            return True

    def _find_conda_env_paths_from_conda(self):
        """Returns a list of path as given by `conda env list --json`.

        Returns empty list, if conda couldn't be called.
        """
        # use the cache, because using conda all the time is slow...
        import subprocess
        import json
        try:
            p = subprocess.Popen(
                ['conda', 'env', 'list', '--json'],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE)
            comm = p.communicate()
            output = comm[0].decode()
            if p.returncode != 0 or len(output) == 0:
                self.log.error("Couldn't call 'conda' to get the environments. "
                               "Output:\n%s" , str(comm))
                return []
        except FileNotFoundError:
            self.log.error("'conda' not found in path.")
            return []
        output = json.loads(output)
        envs = output["envs"]
        #self.log.info("Found the following kernels from conda: %s", ", ".join(envs))
        return envs

    def _find_conda_env_path_from_config(self):
        """Returns a list of env paths as by using the configured conda env basedir"""
        return self._find_env_paths_in_basedirs(self.conda_env_dirs)

    def _find_virtualenv_env_path_from_config(self):
        """Returns a list of env paths as by using the configured virtualenv basedir"""
        return self._find_env_paths_in_basedirs(self.virtualenv_env_dirs)

    def _validate_ipython_path(self, env_path):
        """Returns whether a ipython executable is in that env"""
        # We validate a dir if it has a ipython executable in one of the dirs
        # which ends up in PATH. The proper way would probably use the ipykernel
        # as ipython itself is no garantee that the env can also be used as a kernel
        # but it is close...
        ipython = "ipython.exe" if platform.system() == "Windows" else "ipython"
        for search_path in ['bin', 'Scripts']:
            ipython_path = pj(env_path, search_path, ipython)
            if os.path.exists(ipython_path):
                return True
        return False

    def _find_env_paths_in_basedirs(self, base_dirs):
        """Returns all potential envs in a basedir"""
        # get potential env path in the base_dirs
        envs = []
        for base_dir in base_dirs:
            envs.extend(glob.glob(pj(os.path.expanduser(base_dir), '*', '')))
        #self.log.info("Found the following kernels from config: %s", ", ".join(venvs))

        # filter out envs which do not contain a ipython (~= ipkernel)
        envs = [env for env in envs if self._validate_ipython_path(env)]

        return envs

    def _convert_to_envs(self, env_paths, nametemplate):
        """Returns a dict of kernel_name -> path

        kernelname is build by using the template.
        """
        envs = {}
        for venv_dir in env_paths:
            venv_name = os.path.split(venv_dir)[1]
            kernel_name = nametemplate.format(venv_name)
            kernel_name = kernel_name.lower()
            if kernel_name in envs:
                self.log.error("Duplicate env kernels: %s would both point to %s and %s. Using the first!", kernel_name, envs[kernel_name], venv_dir)
                continue
            if self.validate_env(kernel_name):
                envs.update({kernel_name: venv_dir})
            else:
                self.log.info("Not considered env kernel (blacklisted): %s", kernel_name)
        return envs

    def _find_conda_envs(self):
        """Returns conda envs as dict of name -> path"""
        paths = self._find_conda_env_path_from_config()
        paths.extend(self._find_conda_env_paths_from_conda())
        paths = list(set(paths))

        # can't use '/' as that results in javascript errors :-/
        # need to use something which matches \w to get logos:
        # https://github.com/jupyter/notebook/issues/853
        templ = "conda_{}"
        return self._convert_to_envs(paths, templ)

    def _find_virtualenv_envs(self):
        """Returns virtualenv envs as dict of name -> path"""
        paths = self._find_virtualenv_env_path_from_config()

        templ = "virtualenv_{}"
        return self._convert_to_envs(paths, templ)

    def find_envs(self):
        """Returns for all envs as dict of name -> path"""

        # This is called much too often and the conda calls are really expensive :-(
        if hasattr(self, "_find_envs_cache"):
            return getattr(self, "_find_envs_cache")

        envs = {}
        envs.update(self._find_conda_envs())
        envs.update(self._find_virtualenv_envs())
        if envs:
            self.log.info("Found the following kernels for environments: %s", ", ".join(list(envs)))
        else:
            self.log.info("Found no kernels from environments!")
        self._find_envs_cache = envs
        return envs

    def _build_kernel_specs(self):
        """Returns the dict of name -> kernel_spec for all environments"""

        # This is called much too often and the conda calls are really expensive :-(
        if hasattr(self, "_build_kernel_specs_cache"):
            return getattr(self, "_build_kernel_specs_cache")

        venv_dirs = self.find_envs()
        kspecs = {}
        if platform.system() == "Windows":
            python_exe_name = "python.exe"
        else:
            python_exe_name = "python"

        for venv_name, venv_dir in venv_dirs.items():
            # conda on windows has python.exe directly in the env
            exe_name = pj(venv_dir, python_exe_name)
            if not os.path.exists(exe_name):
                exe_name = pj(venv_dir, "bin", python_exe_name)
            kspec_dict =  {"argv": [exe_name,
                                    "-m",
                                    "IPython.kernel",
                                    "-f",
                                    "{connection_file}"],
                           "language": "python",
                           "display_name": self.display_name_template.format(venv_name),
                           "env": {}}
            # This should probably use self.kernel_spec_class instead of the direct class
            kspecs.update({venv_name: KernelSpec(resource_dir=RESOURCES, **kspec_dict)})
        self._build_kernel_specs_cache = kspecs
        return kspecs

    def find_kernel_specs_for_envs(self):
        """Returns a dict mapping kernel names to resource directories."""
        # as the envs do not have logos (=resources) included, add in the
        # normal ones for python

        envs = self.find_envs()
        ret = {}
        for name in envs:
            ret[name] = RESOURCES
        return ret

    def find_kernel_specs(self):
        """Returns a dict mapping kernel names to resource directories."""
        # let real installed kernels overwrite envs with the same name:
        # this is the same order as the get_kernel_spec way, which also prefers
        # kernels from the jupyter dir over env kernels.
        specs = self.find_kernel_specs_for_envs()
        specs.update(super(EnvironmentKernelSpecManager, self).find_kernel_specs())
        return specs

    def get_kernel_spec(self, kernel_name):
        """Returns a :class:`KernelSpec` instance for the given kernel_name.

        Raises :exc:`NoSuchKernel` if the given kernel name is not found.
        """
        try:
            return super(EnvironmentKernelSpecManager, self).get_kernel_spec(kernel_name)
        except (NoSuchKernel, FileNotFoundError):
            venv_kernel_name = kernel_name.lower()
            specs = self._build_kernel_specs()
            if venv_kernel_name in specs:
                return specs[venv_kernel_name]
            else:
                raise NoSuchKernel(kernel_name)
<|endoftext|>


<commit_before># This file is part of Bika LIMS
#
# Copyright 2011-2016 by it's authors.
# Some rights reserved. See LICENSE.txt, AUTHORS.txt.

from AccessControl import ClassSecurityInfo, Unauthorized
from Products.ATExtensions.Extensions.utils import makeDisplayList
from Products.ATExtensions.ateapi import RecordField, RecordsField
from Products.Archetypes.Registry import registerField
from Products.Archetypes.public import *
from Products.CMFCore.utils import getToolByName
from Products.CMFEditions.ArchivistTool import ArchivistRetrieveError
from Products.validation import validation
from Products.validation.validators.RegexValidator import RegexValidator
import sys
from Products.CMFEditions.Permissions import SaveNewVersion
from Products.CMFEditions.Permissions import AccessPreviousVersions
from Products.Archetypes.config import REFERENCE_CATALOG
from bika.lims import bikaMessageFactory as _
from bika.lims.utils import t
from bika.lims import logger
from bika.lims.utils import to_utf8

class HistoryAwareReferenceField(ReferenceField):
    """ Version aware references.

    Uses instance.reference_versions[uid] to record uid.version_id,
    to pin this reference to a specific version.

    The 'auto_update_backrefs' is a list of reference relationship names
    which will automatically be updated when this object's version is
    incremented:  https://github.com/bikalabs/Bika-LIMS/issues/84

    """
    security = ClassSecurityInfo()

    security.declarePrivate('set')

    def set(self, instance, value, **kwargs):
        """ Mutator. """

        rc = getToolByName(instance, REFERENCE_CATALOG)
        targetUIDs = [ref.targetUID for ref in
                      rc.getReferences(instance, self.relationship)]

        # empty value
        if not value:
            value = ()
        # list with one empty item
        if type(value) in (list, tuple) and len(value) == 1 and not value[0]:
            value = ()

        if not value and not targetUIDs:
            return

        if not isinstance(value, (list, tuple)):
            value = value,
        elif not self.multiValued and len(value) > 1:
            raise ValueError("Multiple values given for single valued field %r" % self)

        ts = getToolByName(instance, "translation_service").translate

        #convert objects to uids
        #convert uids to objects
        uids = []
        targets = {}
        for v in value:
            if isinstance(v, basestring):
                uids.append(v)
                targets[v] = rc.lookupObject(v)
            elif hasattr(v, 'UID'):
                target_uid = callable(v.UID) and v.UID() or v.UID
                uids.append(target_uid)
                targets[target_uid] = v
            else:
                logger.info("Target has no UID: %s/%s" % (v, value))

        sub = [t for t in targetUIDs if t not in uids]
        add = [v for v in uids if v and v not in targetUIDs]

        newuids = [t for t in list(targetUIDs) + list(uids) if t not in sub]
        for uid in newuids:
            # update version_id of all existing references that aren't
            # about to be removed anyway (contents of sub)
            version_id = hasattr(targets[uid], 'version_id') and \
                       targets[uid].version_id or None
            if version_id is None:
                # attempt initial save of unversioned targets
                pr = getToolByName(instance, 'portal_repository')
                if pr.isVersionable(targets[uid]):
                    pr.save(obj=targets[uid],
                            comment=to_utf8(ts(_("Initial revision"))))
            if not hasattr(instance, 'reference_versions'):
                instance.reference_versions = {}
            if not hasattr(targets[uid], 'version_id'):
                targets[uid].version_id = None
            instance.reference_versions[uid] = targets[uid].version_id

        # tweak keyword arguments for addReference
        addRef_kw = kwargs.copy()
        addRef_kw.setdefault('referenceClass', self.referenceClass)
        if 'schema' in addRef_kw:
            del addRef_kw['schema']
        for uid in add:
            __traceback_info__ = (instance, uid, value, targetUIDs)
            # throws IndexError if uid is invalid
            rc.addReference(instance, uid, self.relationship, **addRef_kw)

        for uid in sub:
            rc.deleteReference(instance, uid, self.relationship)

        if self.referencesSortable:
            if not hasattr(aq_base(instance), 'at_ordered_refs'):
                instance.at_ordered_refs = {}

            instance.at_ordered_refs[self.relationship] = \
                tuple(filter(None, uids))

        if self.callStorageOnSet:
            #if this option is set the reference fields's values get written
            #to the storage even if the reference field never use the storage
            #e.g. if i want to store the reference UIDs into an SQL field
            ObjectField.set(self, instance, self.getRaw(instance), **kwargs)

    security.declarePrivate('get')

    def get(self, instance, aslist=False, **kwargs):
        """get() returns the list of objects referenced under the relationship.
        """
        uc = getToolByName(instance, "uid_catalog")

        try:
            res = instance.getRefs(relationship=self.relationship)
        except:
            pass

        pr = getToolByName(instance, 'portal_repository')

        rd = {}
        for r in res:
            if r is None:
                continue
            uid = r.UID()
            r = uc(UID=uid)[0].getObject()
            if hasattr(instance, 'reference_versions') and \
               hasattr(r, 'version_id') and \
               uid in instance.reference_versions and \
               instance.reference_versions[uid] != r.version_id and \
               r.version_id is not None:

                version_id = instance.reference_versions[uid]
                try:
                    o = pr._retrieve(r,
                                     selector=version_id,
                                     preserve=(),
                                     countPurged=True).object
                except ArchivistRetrieveError:
                    o = r
            else:
                o = r
            rd[uid] = o

        # singlevalued ref fields return only the object, not a list,
        # unless explicitely specified by the aslist option

        if not self.multiValued:
            if len(rd) > 1:
                log("%s references for non multivalued field %s of %s" %
                    (len(rd), self.getName(), instance))
            if not aslist:
                if rd:
                    rd = [rd[uid] for uid in rd.keys()][0]
                else:
                    rd = None

        if not self.referencesSortable or not hasattr(aq_base(instance),
                                                      'at_ordered_refs'):
            if isinstance(rd, dict):
                return [rd[uid] for uid in rd.keys()]
            else:
                return rd
        refs = instance.at_ordered_refs
        order = refs[self.relationship]

        return [rd[uid] for uid in order if uid in rd.keys()]

registerField(HistoryAwareReferenceField,
              title="History Aware Reference",
              description="",
              )
<commit_msg>Fix test's "AttributeError: uid_catalog"<commit_after># This file is part of Bika LIMS
#
# Copyright 2011-2016 by it's authors.
# Some rights reserved. See LICENSE.txt, AUTHORS.txt.

from AccessControl import ClassSecurityInfo, Unauthorized
from Products.ATExtensions.Extensions.utils import makeDisplayList
from Products.ATExtensions.ateapi import RecordField, RecordsField
from Products.Archetypes.Registry import registerField
from Products.Archetypes.public import *
from Products.CMFCore.utils import getToolByName
from Products.CMFEditions.ArchivistTool import ArchivistRetrieveError
from Products.validation import validation
from Products.validation.validators.RegexValidator import RegexValidator
import sys
from Products.CMFEditions.Permissions import SaveNewVersion
from Products.CMFEditions.Permissions import AccessPreviousVersions
from Products.Archetypes.config import REFERENCE_CATALOG
from bika.lims import bikaMessageFactory as _
from bika.lims.utils import t
from bika.lims import logger
from bika.lims.utils import to_utf8

class HistoryAwareReferenceField(ReferenceField):
    """ Version aware references.

    Uses instance.reference_versions[uid] to record uid.version_id,
    to pin this reference to a specific version.

    The 'auto_update_backrefs' is a list of reference relationship names
    which will automatically be updated when this object's version is
    incremented:  https://github.com/bikalabs/Bika-LIMS/issues/84

    """
    security = ClassSecurityInfo()

    security.declarePrivate('set')

    def set(self, instance, value, **kwargs):
        """ Mutator. """

        rc = getToolByName(instance, REFERENCE_CATALOG)
        targetUIDs = [ref.targetUID for ref in
                      rc.getReferences(instance, self.relationship)]

        # empty value
        if not value:
            value = ()
        # list with one empty item
        if type(value) in (list, tuple) and len(value) == 1 and not value[0]:
            value = ()

        if not value and not targetUIDs:
            return

        if not isinstance(value, (list, tuple)):
            value = value,
        elif not self.multiValued and len(value) > 1:
            raise ValueError("Multiple values given for single valued field %r" % self)

        ts = getToolByName(instance, "translation_service").translate

        #convert objects to uids
        #convert uids to objects
        uids = []
        targets = {}
        for v in value:
            if isinstance(v, basestring):
                uids.append(v)
                targets[v] = rc.lookupObject(v)
            elif hasattr(v, 'UID'):
                target_uid = callable(v.UID) and v.UID() or v.UID
                uids.append(target_uid)
                targets[target_uid] = v
            else:
                logger.info("Target has no UID: %s/%s" % (v, value))

        sub = [t for t in targetUIDs if t not in uids]
        add = [v for v in uids if v and v not in targetUIDs]

        newuids = [t for t in list(targetUIDs) + list(uids) if t not in sub]
        for uid in newuids:
            # update version_id of all existing references that aren't
            # about to be removed anyway (contents of sub)
            version_id = hasattr(targets[uid], 'version_id') and \
                       targets[uid].version_id or None
            if version_id is None:
                # attempt initial save of unversioned targets
                pr = getToolByName(instance, 'portal_repository')
                if pr.isVersionable(targets[uid]):
                    pr.save(obj=targets[uid],
                            comment=to_utf8(ts(_("Initial revision"))))
            if not hasattr(instance, 'reference_versions'):
                instance.reference_versions = {}
            if not hasattr(targets[uid], 'version_id'):
                targets[uid].version_id = None
            instance.reference_versions[uid] = targets[uid].version_id

        # tweak keyword arguments for addReference
        addRef_kw = kwargs.copy()
        addRef_kw.setdefault('referenceClass', self.referenceClass)
        if 'schema' in addRef_kw:
            del addRef_kw['schema']
        for uid in add:
            __traceback_info__ = (instance, uid, value, targetUIDs)
            # throws IndexError if uid is invalid
            rc.addReference(instance, uid, self.relationship, **addRef_kw)

        for uid in sub:
            rc.deleteReference(instance, uid, self.relationship)

        if self.referencesSortable:
            if not hasattr(aq_base(instance), 'at_ordered_refs'):
                instance.at_ordered_refs = {}

            instance.at_ordered_refs[self.relationship] = \
                tuple(filter(None, uids))

        if self.callStorageOnSet:
            #if this option is set the reference fields's values get written
            #to the storage even if the reference field never use the storage
            #e.g. if i want to store the reference UIDs into an SQL field
            ObjectField.set(self, instance, self.getRaw(instance), **kwargs)

    security.declarePrivate('get')

    def get(self, instance, aslist=False, **kwargs):
        """get() returns the list of objects referenced under the relationship.
        """
        try:
            uc = getToolByName(instance, "uid_catalog")
        except AttributeError as err:
            logger.error("AttributeError: {0}".format(err))
            return []

        try:
            res = instance.getRefs(relationship=self.relationship)
        except:
            pass

        pr = getToolByName(instance, 'portal_repository')

        rd = {}
        for r in res:
            if r is None:
                continue
            uid = r.UID()
            r = uc(UID=uid)[0].getObject()
            if hasattr(instance, 'reference_versions') and \
               hasattr(r, 'version_id') and \
               uid in instance.reference_versions and \
               instance.reference_versions[uid] != r.version_id and \
               r.version_id is not None:

                version_id = instance.reference_versions[uid]
                try:
                    o = pr._retrieve(r,
                                     selector=version_id,
                                     preserve=(),
                                     countPurged=True).object
                except ArchivistRetrieveError:
                    o = r
            else:
                o = r
            rd[uid] = o

        # singlevalued ref fields return only the object, not a list,
        # unless explicitely specified by the aslist option

        if not self.multiValued:
            if len(rd) > 1:
                log("%s references for non multivalued field %s of %s" %
                    (len(rd), self.getName(), instance))
            if not aslist:
                if rd:
                    rd = [rd[uid] for uid in rd.keys()][0]
                else:
                    rd = None

        if not self.referencesSortable or not hasattr(aq_base(instance),
                                                      'at_ordered_refs'):
            if isinstance(rd, dict):
                return [rd[uid] for uid in rd.keys()]
            else:
                return rd
        refs = instance.at_ordered_refs
        order = refs[self.relationship]

        return [rd[uid] for uid in order if uid in rd.keys()]

registerField(HistoryAwareReferenceField,
              title="History Aware Reference",
              description="",
              )
<|endoftext|>