<commit_before>"""Forms for the ``paypal_express_checkout`` app."""
import httplib
import logging
import urllib
import urllib2
import urlparse

from django import forms
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import Http404
from django.shortcuts import redirect
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _

from .constants import PAYMENT_STATUS, PAYPAL_DEFAULTS
from .models import (
    Item,
    PaymentTransaction,
    PaymentTransactionError,
    PurchasedItem,
)
from .settings import API_URL, LOGIN_URL

logger = logging.getLogger(__name__)

class PayPalFormMixin(object):
    """Common methods for the PayPal forms."""
    def call_paypal(self, post_data):
        """
        Gets the PayPal API URL from the settings and posts ``post_data``.

        :param post_data: The full post data for PayPal containing all needed
          information for the current transaction step.

        """
        try:
            response = urllib2.urlopen(
                API_URL, data=urllib.urlencode(post_data))
        except (
                urllib2.HTTPError,
                urllib2.URLError,
                httplib.HTTPException), ex:
            self.log_error(ex)
        else:
            parsed_response = urlparse.parse_qs(response.read())
            return parsed_response

    def log_error(self, error_message, transaction=None):
        """
        Saves error information as a ``PaymentTransactionError`` object.

        :param error_message: The message of the exception or response string
          from PayPal.

        """
        payment_error = PaymentTransactionError()
        payment_error.user = self.user
        payment_error.response = error_message
        payment_error.transaction = transaction
        payment_error.save()
        return payment_error

class DoExpressCheckoutForm(PayPalFormMixin, forms.Form):
    """
    Takes the input from the ``DoExpressCheckoutView``, validates it and
    takes care of the PayPal API operations.

    """
    token = forms.CharField()

    payerID = forms.CharField()

    def __init__(self, user, *args, **kwargs):
        super(DoExpressCheckoutForm, self).__init__(*args, **kwargs)
        try:
            self.transaction = PaymentTransaction.objects.get(
                user=user, transaction_id=self.data['token'])
        except PaymentTransaction.DoesNotExist:
            raise Http404

    def get_post_data(self):
        """Creates the post data dictionary to send to PayPal."""
        post_data = PAYPAL_DEFAULTS
        post_data.update({
            'METHOD': 'DoExpressCheckoutPayment',
            'TOKEN': self.transaction.transaction_id,
            'PAYERID': self.data['payerID'],
            'PAYMENTREQUEST_0_AMT': self.transaction.value,
            'PAYMENTREQUEST_0_NOTIFYURL': settings.HOSTNAME + reverse(
                'ipn_listener'),
        })
        return post_data

    def do_checkout(self):
        """Calls PayPal to make the 'DoExpressCheckoutPayment' procedure."""
        post_data = self.get_post_data()
        parsed_response = self.call_paypal(post_data)
        if parsed_response.get('ACK')[0] == 'Success':
            transaction_id = parsed_response.get(
                'PAYMENTINFO_0_TRANSACTIONID')[0]
            self.transaction.transaction_id = transaction_id
            self.transaction.status = PAYMENT_STATUS['pending']
            self.transaction.save()
            return redirect(reverse('paypal_success'))
        elif parsed_response.get('ACK')[0] == 'Failure':
            self.transaction.status = PAYMENT_STATUS['error']
            self.transaction.save()
            self.log_error(parsed_response, self.transaction)
            return redirect(reverse('paypal_error'))

class SetExpressCheckoutFormMixin(PayPalFormMixin, forms.Form):
    """
    Base form class for all forms invoking the ``SetExpressCheckout`` PayPal
    API operation, providing the general method skeleton.

    Also this is to be used to construct custom forms.

    """
    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(SetExpressCheckoutFormMixin, self).__init__(*args, **kwargs)

    def get_content_object(self):
        """
        Can be overridden to return a different content object for the
        PaymentTransaction model.

        This is useful if you want e.g. have one of your models assigned to the
        transaction for easier identification.

        """
        # TODO for now it should return the user, although I know, that the
        # user is already present in the user field of the PaymentTransaction
        # model.
        # Maybe we can remove the user field safely in exchange for the generic
        # relation only.
        return self.user

    def get_item(self):
        """Obsolete. Just implement ``get_items_and_quantities``."""
        raise NotImplemented

    def get_quantity(self):
        """Obsolete. Just implement ``get_items_and_quantities``."""
        raise NotImplemented

    def get_items_and_quantities(self):
        """
        Returns the items and quantities.

        Should return a list of tuples: ``[(item, quantity), ]``

        """
        logger.warning(
            'Deprecation warning: Please implement get_items_and_quantities on'
            ' your SetExpressCheckoutForm. Do not use get_item and'
            ' get_quantity any more.')
        return [(self.get_item(), self.get_quantity()), ]

    def get_post_data(self, item_quantity_list):
        """Creates the post data dictionary to send to PayPal."""
        post_data = PAYPAL_DEFAULTS
        total_value = 0
        item_index = 0
        for item, quantity in item_quantity_list:
            if not quantity:
                # If a user chose quantity 0, we don't include it
                continue
            total_value += item.value * quantity
            post_data.update({
                'L_PAYMENTREQUEST_0_NAME{0}'.format(
                    item_index): item.name,
                'L_PAYMENTREQUEST_0_DESC{0}'.format(
                    item_index): item.description,
                'L_PAYMENTREQUEST_0_AMT{0}'.format(
                    item_index): item.value,
                'L_PAYMENTREQUEST_0_QTY{0}'.format(
                    item_index): quantity,
            })
            item_index += 1

        post_data.update({
            'METHOD': 'SetExpressCheckout',
            'PAYMENTREQUEST_0_AMT': total_value,
            'PAYMENTREQUEST_0_ITEMAMT': total_value,
            'RETURNURL': settings.HOSTNAME + reverse(
                'paypal_confirm', kwargs=self.get_url_kwargs()),
            'CANCELURL': settings.HOSTNAME + reverse(
                'paypal_canceled', kwargs=self.get_url_kwargs()),
        })
        return post_data

    def get_url_kwargs(self):
        """Provide additional url kwargs, by overriding this method."""
        return {}

    def set_checkout(self):
        """
        Calls PayPal to make the 'SetExpressCheckout' procedure.

        :param items: A list of ``Item`` objects.

        """
        item_quantity_list = self.get_items_and_quantities()
        post_data = self.get_post_data(item_quantity_list)

        # making the post to paypal and handling the results
        parsed_response = self.call_paypal(post_data)
        if parsed_response.get('ACK')[0] == 'Success':
            token = parsed_response.get('TOKEN')[0]
            transaction = PaymentTransaction(
                user=self.user,
                date=now(),
                transaction_id=token,
                value=post_data['PAYMENTREQUEST_0_AMT'],
                status=PAYMENT_STATUS['checkout'],
                content_object=self.get_content_object(),
            )
            transaction.save()
            for item, quantity in item_quantity_list:
                if not quantity:
                    continue
                item_kwargs = {
                    'user': self.user,
                    'transaction': transaction,
                    'quantity': quantity,
                }
                if item.pk:
                    item_kwargs.update({'item': item, })
                PurchasedItem.objects.create(**item_kwargs)
            return redirect(LOGIN_URL + token)
        elif parsed_response.get('ACK')[0] == 'Failure':
            self.log_error(parsed_response)
            return redirect(reverse('paypal_error'))

class SetExpressCheckoutItemForm(SetExpressCheckoutFormMixin):
    """
    Takes the input from the ``SetExpressCheckoutView``, validates it and
    takes care of the PayPal API operations.

    """
    item = forms.ModelChoiceField(
        queryset=Item.objects.all(),
        empty_label=None,
        label=_('Item'),
    )

    quantity = forms.IntegerField(
        label=_('Quantity'),
    )

    def get_item(self):
        """Keeping this for backwards compatibility."""
        return self.cleaned_data.get('item')

    def get_quantity(self):
        """Keeping this for backwards compatibility."""
        return self.cleaned_data.get('quantity')

    def get_items_and_quantities(self):
        """
        Returns the items and quantities.

        Should return a list of tuples.

        """
        return [
            (self.get_item(), self.get_quantity()),
        ]
<commit_msg>Corrected status in DoExpressCheckoutForm.<commit_after>"""Forms for the ``paypal_express_checkout`` app."""
import httplib
import logging
import urllib
import urllib2
import urlparse

from django import forms
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import Http404
from django.shortcuts import redirect
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _

from .constants import PAYMENT_STATUS, PAYPAL_DEFAULTS
from .models import (
    Item,
    PaymentTransaction,
    PaymentTransactionError,
    PurchasedItem,
)
from .settings import API_URL, LOGIN_URL

logger = logging.getLogger(__name__)

class PayPalFormMixin(object):
    """Common methods for the PayPal forms."""
    def call_paypal(self, post_data):
        """
        Gets the PayPal API URL from the settings and posts ``post_data``.

        :param post_data: The full post data for PayPal containing all needed
          information for the current transaction step.

        """
        try:
            response = urllib2.urlopen(
                API_URL, data=urllib.urlencode(post_data))
        except (
                urllib2.HTTPError,
                urllib2.URLError,
                httplib.HTTPException), ex:
            self.log_error(ex)
        else:
            parsed_response = urlparse.parse_qs(response.read())
            return parsed_response

    def log_error(self, error_message, transaction=None):
        """
        Saves error information as a ``PaymentTransactionError`` object.

        :param error_message: The message of the exception or response string
          from PayPal.

        """
        payment_error = PaymentTransactionError()
        payment_error.user = self.user
        payment_error.response = error_message
        payment_error.transaction = transaction
        payment_error.save()
        return payment_error

class DoExpressCheckoutForm(PayPalFormMixin, forms.Form):
    """
    Takes the input from the ``DoExpressCheckoutView``, validates it and
    takes care of the PayPal API operations.

    """
    token = forms.CharField()

    payerID = forms.CharField()

    def __init__(self, user, *args, **kwargs):
        super(DoExpressCheckoutForm, self).__init__(*args, **kwargs)
        try:
            self.transaction = PaymentTransaction.objects.get(
                user=user, transaction_id=self.data['token'])
        except PaymentTransaction.DoesNotExist:
            raise Http404

    def get_post_data(self):
        """Creates the post data dictionary to send to PayPal."""
        post_data = PAYPAL_DEFAULTS
        post_data.update({
            'METHOD': 'DoExpressCheckoutPayment',
            'TOKEN': self.transaction.transaction_id,
            'PAYERID': self.data['payerID'],
            'PAYMENTREQUEST_0_AMT': self.transaction.value,
            'PAYMENTREQUEST_0_NOTIFYURL': settings.HOSTNAME + reverse(
                'ipn_listener'),
        })
        return post_data

    def do_checkout(self):
        """Calls PayPal to make the 'DoExpressCheckoutPayment' procedure."""
        post_data = self.get_post_data()
        parsed_response = self.call_paypal(post_data)
        if parsed_response.get('ACK')[0] == 'Success':
            transaction_id = parsed_response.get(
                'PAYMENTINFO_0_TRANSACTIONID')[0]
            self.transaction.transaction_id = transaction_id
            self.transaction.status = PAYMENT_STATUS['pending']
            self.transaction.save()
            return redirect(reverse('paypal_success'))
        elif parsed_response.get('ACK')[0] == 'Failure':
            self.transaction.status = PAYMENT_STATUS['canceled']
            self.transaction.save()
            self.log_error(parsed_response, self.transaction)
            return redirect(reverse('paypal_error'))

class SetExpressCheckoutFormMixin(PayPalFormMixin, forms.Form):
    """
    Base form class for all forms invoking the ``SetExpressCheckout`` PayPal
    API operation, providing the general method skeleton.

    Also this is to be used to construct custom forms.

    """
    def __init__(self, user, *args, **kwargs):
        self.user = user
        super(SetExpressCheckoutFormMixin, self).__init__(*args, **kwargs)

    def get_content_object(self):
        """
        Can be overridden to return a different content object for the
        PaymentTransaction model.

        This is useful if you want e.g. have one of your models assigned to the
        transaction for easier identification.

        """
        # TODO for now it should return the user, although I know, that the
        # user is already present in the user field of the PaymentTransaction
        # model.
        # Maybe we can remove the user field safely in exchange for the generic
        # relation only.
        return self.user

    def get_item(self):
        """Obsolete. Just implement ``get_items_and_quantities``."""
        raise NotImplemented

    def get_quantity(self):
        """Obsolete. Just implement ``get_items_and_quantities``."""
        raise NotImplemented

    def get_items_and_quantities(self):
        """
        Returns the items and quantities.

        Should return a list of tuples: ``[(item, quantity), ]``

        """
        logger.warning(
            'Deprecation warning: Please implement get_items_and_quantities on'
            ' your SetExpressCheckoutForm. Do not use get_item and'
            ' get_quantity any more.')
        return [(self.get_item(), self.get_quantity()), ]

    def get_post_data(self, item_quantity_list):
        """Creates the post data dictionary to send to PayPal."""
        post_data = PAYPAL_DEFAULTS
        total_value = 0
        item_index = 0
        for item, quantity in item_quantity_list:
            if not quantity:
                # If a user chose quantity 0, we don't include it
                continue
            total_value += item.value * quantity
            post_data.update({
                'L_PAYMENTREQUEST_0_NAME{0}'.format(
                    item_index): item.name,
                'L_PAYMENTREQUEST_0_DESC{0}'.format(
                    item_index): item.description,
                'L_PAYMENTREQUEST_0_AMT{0}'.format(
                    item_index): item.value,
                'L_PAYMENTREQUEST_0_QTY{0}'.format(
                    item_index): quantity,
            })
            item_index += 1

        post_data.update({
            'METHOD': 'SetExpressCheckout',
            'PAYMENTREQUEST_0_AMT': total_value,
            'PAYMENTREQUEST_0_ITEMAMT': total_value,
            'RETURNURL': settings.HOSTNAME + reverse(
                'paypal_confirm', kwargs=self.get_url_kwargs()),
            'CANCELURL': settings.HOSTNAME + reverse(
                'paypal_canceled', kwargs=self.get_url_kwargs()),
        })
        return post_data

    def get_url_kwargs(self):
        """Provide additional url kwargs, by overriding this method."""
        return {}

    def set_checkout(self):
        """
        Calls PayPal to make the 'SetExpressCheckout' procedure.

        :param items: A list of ``Item`` objects.

        """
        item_quantity_list = self.get_items_and_quantities()
        post_data = self.get_post_data(item_quantity_list)

        # making the post to paypal and handling the results
        parsed_response = self.call_paypal(post_data)
        if parsed_response.get('ACK')[0] == 'Success':
            token = parsed_response.get('TOKEN')[0]
            transaction = PaymentTransaction(
                user=self.user,
                date=now(),
                transaction_id=token,
                value=post_data['PAYMENTREQUEST_0_AMT'],
                status=PAYMENT_STATUS['checkout'],
                content_object=self.get_content_object(),
            )
            transaction.save()
            for item, quantity in item_quantity_list:
                if not quantity:
                    continue
                item_kwargs = {
                    'user': self.user,
                    'transaction': transaction,
                    'quantity': quantity,
                }
                if item.pk:
                    item_kwargs.update({'item': item, })
                PurchasedItem.objects.create(**item_kwargs)
            return redirect(LOGIN_URL + token)
        elif parsed_response.get('ACK')[0] == 'Failure':
            self.log_error(parsed_response)
            return redirect(reverse('paypal_error'))

class SetExpressCheckoutItemForm(SetExpressCheckoutFormMixin):
    """
    Takes the input from the ``SetExpressCheckoutView``, validates it and
    takes care of the PayPal API operations.

    """
    item = forms.ModelChoiceField(
        queryset=Item.objects.all(),
        empty_label=None,
        label=_('Item'),
    )

    quantity = forms.IntegerField(
        label=_('Quantity'),
    )

    def get_item(self):
        """Keeping this for backwards compatibility."""
        return self.cleaned_data.get('item')

    def get_quantity(self):
        """Keeping this for backwards compatibility."""
        return self.cleaned_data.get('quantity')

    def get_items_and_quantities(self):
        """
        Returns the items and quantities.

        Should return a list of tuples.

        """
        return [
            (self.get_item(), self.get_quantity()),
        ]
<|endoftext|>


<commit_before># -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the HERA Collaboration
# Licensed under the BSD License.

"""Manage background tasks.

These are of course chained to the lifetime of the server process, so we lose
track of them all if the server dies, which of course could happen without
notice at any time.

This system requires Tornado.

"""

__all__ = str('''
BackgroundTask
submit_background_task
register_background_task_reporter
get_unfinished_task_count
''').split()

import time
import asyncio

from tornado.ioloop import IOLoop
from flask import flash, redirect, render_template, url_for

from . import app, db, logger
from .dbutil import NotNull
from .webutil import ServerError, json_api, login_required, optional_arg, required_arg

class BackgroundTask(object):
    """A class implementing a background task.

    Instances of this task are also used by the Librarian to keep track of its
    background tasks. Override the `thread_function` and `wrapup_function`
    methods to implement your actual task: the `thread_function` is called in
    a background thread and takes a long time, but cannot access the database
    or web services. The `wrapup_function` is fed the results of
    `thread_function` and can access the database, but cannot take very long
    to run.

    You need to submit the task to TaskManager for it to actually run!

    """
    _manager = None
    desc = 'unset description'
    submit_time = None
    start_time = None
    finish_time = None
    exception = None

    def __str__(self):
        return self.desc

    def thread_function(self):
        raise NotImplementedError()

    def wrapup_function(self, thread_retval, thread_exc):
        """This function is passed two arguments: the return value from the
        `thread_function` function, and an exception that it may have raised.
        If the exception is not None, the thread_function failed and
        appropriate action should be taken.

        """
        raise NotImplementedError()

    @property
    def runtime(self):
        if self.start_time is None:
            return float('NaN')
        if self.finish_time is None:
            return time.time() - self.start_time
        return self.finish_time - self.start_time

    @property
    def wait_time(self):
        if self.start_time is None:
            return time.time() - self.submit_time
        return self.start_time - self.submit_time

    @property
    def time_since_completed(self):
        if self.finish_time is None:
            return float('NaN')
        return time.time() - self.finish_time

    @property
    def outcome_str(self):
        if self.exception is None:
            return 'success'
        return str(self.exception)

MAX_PURGE_FREQUENCY = 60  # seconds
MIN_TASK_LIST_LENGTH = 20  # don't purge tasks if more than these are left
TASK_LINGER_TIME = 600  # seconds

async def _thread_wrapper(task):
    task.start_time = time.time()

    try:
        retval = task.thread_function()
        exc = None
    except Exception as e:
        retval = None
        exc = e

    task.finish_time = time.time()
    task.exception = exc
    IOLoop.current().add_callback(_wrapup_wrapper, task, retval, exc)

async def _wrapup_wrapper(task, thread_retval, thread_exc):
    try:
        task.wrapup_function(thread_retval, thread_exc)
    except Exception as e:
        logger.warn('exception in %s wrapup function: %s', task, e)
        task.exception = thread_exc = e

    # We let the task linger in task list for a little while so that it's
    # possible to review historical activity. But while we're here, see if
    # there's anything to purge.
    the_task_manager._maybe_purge_tasks()

class TaskManager(object):
    tasks = None
    """This is a list of all tasks that are pending, in processing, or have exited
    recently. Eventually we purge them but it's useful to be able to see the
    server's recent activity.

    """
    worker_pool = None
    """A ThreadPool of workers that can execute the background tasks.

    """
    last_purge = 0
    """The last time that the `tasks` list was purged."""

    def __init__(self):
        self.tasks = []
        self.last_purge = time.time()

    def _maybe_purge_tasks(self):
        now = time.time()

        if now - self.last_purge < MAX_PURGE_FREQUENCY:
            # Don't purge more frequently than every minute.
            return

        self.last_purge = now

        if len(self.tasks) <= MIN_TASK_LIST_LENGTH:
            # Don't bother purging if there aren't more than this many tasks
            # listed.
            return

        self.tasks = [t for t in self.tasks
                      if (t.finish_time is None or
                          (now - t.finish_time) < TASK_LINGER_TIME)]

    async def submit(self, task):
        """Submit a task to be run in the background.

        It may not be launched immediately if there are a lot of background
        tasks to deal with.

        apply_async() returns a result object, but we're a web service so we
        can't wait around to see what it is. Instead we run the function in a
        wrapper that uses Tornado's infrastructure to let the main thread know
        what happened to it.

        """
        assert task._manager is None, 'may not submit task to multiple managers'

        self._maybe_purge_tasks()

        task._manager = self

        self.tasks.append(task)
        await _thread_wrapper(task)

the_task_manager = TaskManager()

def submit_background_task(task):
    return IOLoop.current().add_callback(the_task_manager.submit, task)

def log_background_task_status():
    active = [t for t in the_task_manager.tasks
              if t.start_time is not None and t.finish_time is None]
    pending = [t for t in the_task_manager.tasks
               if t.start_time is None]
    finished = [t for t in the_task_manager.tasks
                if t.finish_time is not None]

    logger.info('%d background tasks: %d active, %d pending, %d finished',
                len(the_task_manager.tasks), len(active),
                len(pending), len(finished))

def register_background_task_reporter():
    """Create a Tornado PeriodicCallback that will periodically report on the
    status of background tasks.

    The timeout for the callback is measured in milliseconds, so we queue an
    evaluation every 3 minutes.

    """
    from tornado import ioloop

    cb = ioloop.PeriodicCallback(log_background_task_status, 60 * 3 * 1000)
    cb.start()
    return cb

def get_unfinished_task_count():
    """Get the number of active or pending background tasks. This function
    is used by the M&C subsystem.

    """
    n = 0

    for t in the_task_manager.tasks:
        if t.finish_time is None:
            n += 1

    return n

# Web user interface

@app.route('/tasks')
@login_required
def tasks():
    the_task_manager._maybe_purge_tasks()

    active = [t for t in the_task_manager.tasks
              if t.start_time is not None and t.finish_time is None]
    pending = [t for t in the_task_manager.tasks
               if t.start_time is None]
    finished = [t for t in the_task_manager.tasks
                if t.finish_time is not None]

    return render_template(
        'task-listing.html',
        title='Tasks',
        active=active,
        pending=pending,
        finished=finished,
    )
<commit_msg>Add task submit time back<commit_after># -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the HERA Collaboration
# Licensed under the BSD License.

"""Manage background tasks.

These are of course chained to the lifetime of the server process, so we lose
track of them all if the server dies, which of course could happen without
notice at any time.

This system requires Tornado.

"""

__all__ = str('''
BackgroundTask
submit_background_task
register_background_task_reporter
get_unfinished_task_count
''').split()

import time
import asyncio

from tornado.ioloop import IOLoop
from flask import flash, redirect, render_template, url_for

from . import app, db, logger
from .dbutil import NotNull
from .webutil import ServerError, json_api, login_required, optional_arg, required_arg

class BackgroundTask(object):
    """A class implementing a background task.

    Instances of this task are also used by the Librarian to keep track of its
    background tasks. Override the `thread_function` and `wrapup_function`
    methods to implement your actual task: the `thread_function` is called in
    a background thread and takes a long time, but cannot access the database
    or web services. The `wrapup_function` is fed the results of
    `thread_function` and can access the database, but cannot take very long
    to run.

    You need to submit the task to TaskManager for it to actually run!

    """
    _manager = None
    desc = 'unset description'
    submit_time = None
    start_time = None
    finish_time = None
    exception = None

    def __str__(self):
        return self.desc

    def thread_function(self):
        raise NotImplementedError()

    def wrapup_function(self, thread_retval, thread_exc):
        """This function is passed two arguments: the return value from the
        `thread_function` function, and an exception that it may have raised.
        If the exception is not None, the thread_function failed and
        appropriate action should be taken.

        """
        raise NotImplementedError()

    @property
    def runtime(self):
        if self.start_time is None:
            return float('NaN')
        if self.finish_time is None:
            return time.time() - self.start_time
        return self.finish_time - self.start_time

    @property
    def wait_time(self):
        if self.start_time is None:
            return time.time() - self.submit_time
        return self.start_time - self.submit_time

    @property
    def time_since_completed(self):
        if self.finish_time is None:
            return float('NaN')
        return time.time() - self.finish_time

    @property
    def outcome_str(self):
        if self.exception is None:
            return 'success'
        return str(self.exception)

MAX_PURGE_FREQUENCY = 60  # seconds
MIN_TASK_LIST_LENGTH = 20  # don't purge tasks if more than these are left
TASK_LINGER_TIME = 600  # seconds

async def _thread_wrapper(task):
    task.start_time = time.time()

    try:
        retval = task.thread_function()
        exc = None
    except Exception as e:
        retval = None
        exc = e

    task.finish_time = time.time()
    task.exception = exc
    IOLoop.current().add_callback(_wrapup_wrapper, task, retval, exc)

async def _wrapup_wrapper(task, thread_retval, thread_exc):
    try:
        task.wrapup_function(thread_retval, thread_exc)
    except Exception as e:
        logger.warn('exception in %s wrapup function: %s', task, e)
        task.exception = thread_exc = e

    # We let the task linger in task list for a little while so that it's
    # possible to review historical activity. But while we're here, see if
    # there's anything to purge.
    the_task_manager._maybe_purge_tasks()

class TaskManager(object):
    tasks = None
    """This is a list of all tasks that are pending, in processing, or have exited
    recently. Eventually we purge them but it's useful to be able to see the
    server's recent activity.

    """
    worker_pool = None
    """A ThreadPool of workers that can execute the background tasks.

    """
    last_purge = 0
    """The last time that the `tasks` list was purged."""

    def __init__(self):
        self.tasks = []
        self.last_purge = time.time()

    def _maybe_purge_tasks(self):
        now = time.time()

        if now - self.last_purge < MAX_PURGE_FREQUENCY:
            # Don't purge more frequently than every minute.
            return

        self.last_purge = now

        if len(self.tasks) <= MIN_TASK_LIST_LENGTH:
            # Don't bother purging if there aren't more than this many tasks
            # listed.
            return

        self.tasks = [t for t in self.tasks
                      if (t.finish_time is None or
                          (now - t.finish_time) < TASK_LINGER_TIME)]

    async def submit(self, task):
        """Submit a task to be run in the background.

        It may not be launched immediately if there are a lot of background
        tasks to deal with.

        apply_async() returns a result object, but we're a web service so we
        can't wait around to see what it is. Instead we run the function in a
        wrapper that uses Tornado's infrastructure to let the main thread know
        what happened to it.

        """
        assert task._manager is None, 'may not submit task to multiple managers'

        self._maybe_purge_tasks()

        task._manager = self

        task.submit_time = time.time()
        self.tasks.append(task)
        await _thread_wrapper(task)

the_task_manager = TaskManager()

def submit_background_task(task):
    return IOLoop.current().add_callback(the_task_manager.submit, task)

def log_background_task_status():
    active = [t for t in the_task_manager.tasks
              if t.start_time is not None and t.finish_time is None]
    pending = [t for t in the_task_manager.tasks
               if t.start_time is None]
    finished = [t for t in the_task_manager.tasks
                if t.finish_time is not None]

    logger.info('%d background tasks: %d active, %d pending, %d finished',
                len(the_task_manager.tasks), len(active),
                len(pending), len(finished))

def register_background_task_reporter():
    """Create a Tornado PeriodicCallback that will periodically report on the
    status of background tasks.

    The timeout for the callback is measured in milliseconds, so we queue an
    evaluation every 3 minutes.

    """
    from tornado import ioloop

    cb = ioloop.PeriodicCallback(log_background_task_status, 60 * 3 * 1000)
    cb.start()
    return cb

def get_unfinished_task_count():
    """Get the number of active or pending background tasks. This function
    is used by the M&C subsystem.

    """
    n = 0

    for t in the_task_manager.tasks:
        if t.finish_time is None:
            n += 1

    return n

# Web user interface

@app.route('/tasks')
@login_required
def tasks():
    the_task_manager._maybe_purge_tasks()

    active = [t for t in the_task_manager.tasks
              if t.start_time is not None and t.finish_time is None]
    pending = [t for t in the_task_manager.tasks
               if t.start_time is None]
    finished = [t for t in the_task_manager.tasks
                if t.finish_time is not None]

    return render_template(
        'task-listing.html',
        title='Tasks',
        active=active,
        pending=pending,
        finished=finished,
    )
<|endoftext|>


<commit_before>import argparse
import datetime
import enum
import logging
import numpy as np
import os
import random
import sys
import tempfile
import tensorflow as tf

from keras.applications.imagenet_utils import preprocess_input
from keras.applications.resnet50 import ResNet50
from keras.callbacks import ModelCheckpoint
from keras.models import Model, Sequential
from keras.layers import Dense, Dropout, Flatten, Input, Reshape
from keras.preprocessing import image

import pelops.const as const
from pelops.datasets.dgcars import DGCarsDataset
from pelops.utils import SetType, setup_custom_logger

# =============================================================================
# Allocate GPU memory so the program does not take over the entire GPU memory
# =============================================================================

def __get_session(gpu_fraction=0.3):
    '''Assume that you have 6GB of GPU memory and want to allocate ~2GB'''

    num_threads = os.environ.get('OMP_NUM_THREADS')
    const.logger.info("num_threads: {}".format(num_threads))
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)

    if num_threads:
        return tf.Session(config=tf.ConfigProto(
            gpu_options=gpu_options, intra_op_parallelism_threads=num_threads))
    else:
        return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

# =============================================================================
# Define convolutional model types
# =============================================================================

class ConvType(enum.Enum):
    ResNet50 = "ResNet50"

# =============================================================================
# Helper functions
# =============================================================================

def __get_resnet50(include_top):
    return ResNet50(
            include_top=include_top,
            weights="imagenet",
            input_tensor=Input(
                shape=(
                    const.img_height,
                    const.img_width,
                    const.img_dimension
                )
            )
    )

def __extract_features(model, generator, set_type):
    feature_dirpath = "./features/"
    const.logger.info("create a feature directory to store saved features: {}".format(feature_dirpath))
    if not os.path.exists(feature_dirpath):
        os.makedirs(feature_dirpath)

    const.logger.info("extract features from cnn")
    const.logger.debug("batch_size: {}".format(generator.batch_size))
    features = model.predict_generator(
        generator,
        generator.batch_size
    )

    time_now = datetime.datetime.now().strftime("%Y%m%d_%H_%M_%S")
    features_filepath = feature_dirpath + "{}_{}_{}_features_{}.npy".format(
        const.dataset_type,
        const.conv_model_type,
        set_type,
        time_now
    )
    const.logger.info("save features to {}".format(features_filepath))
    np.save(open(features_filepath, "wb"), features)

    return features, features_filepath

def __load_features(feature_filepath):
    return np.load(open(feature_filepath, "rb"))

def __create_checkpoints(which_model):
    # TODO: something is wrong with this
    checkpoint_dirpath = "./checkpoints/"
    const.logger.info("create a checkpoint directory to store saved checkpoints: {}".format(checkpoint_dirpath))
    if not os.path.exists(checkpoint_dirpath):
        os.makedirs(checkpoint_dirpath)

    checkpoint_filepath = \
        checkpoint_dirpath + \
        "{}_{}_features_".format(const.dataset_type, which_model) + \
        "{epoch:02d}.npy"

    checkpoint = ModelCheckpoint(
        filepath=checkpoint_filepath,
        monitor="val_acc",
        save_best_only=True,
        mode="max",
        save_weights_only=False,
        verbose=2,
    )

    return [checkpoint]

def __create_generator_from_features(features, generator):

    nb_sample = generator.nb_sample
    nb_class = generator.nb_class
    batch_size = generator.batch_size

    # create labels for the features
    count = 0
    labels = np.zeros((batch_size, nb_class))
    for i, class_index in zip(range(0, batch_size), generator.classes):
        labels[i][class_index] = 1

    # create generator with features and labels
    while True:
        for i in range(int(nb_sample/batch_size)):
            x = features[i * batch_size: (i+1) * batch_size]
            y = labels[i * batch_size: (i+1) * batch_size]
            yield x, y

# =============================================================================
# Main
# =============================================================================

def main(args):
    # setup logging
    const.logger = setup_custom_logger(__name__)

    # allocate gpu memory
    # TODO: causes memory error when run
    #session_number = __get_session()
    #const.logger.info("allocate GPU memory, session number: {}".format(session_number))

    # extract arguments from command line
    const.train_dir_path = args.train_dir_path
    const.val_dir_path = args.val_dir_path
    const.train_features_path = args.train_features_path
    const.val_features_path = args.val_features_path
    const.dataset_type = args.dataset_type
    const.conv_model_type = args.conv_model_type
    const.nb_epoch = args.nb_epoch
    const.dropout_rate = args.dropout_rate
    const.seed = args.seed

    # initialize constants
    if const.conv_model_type == ConvType.ResNet50.value:
        # ResNet50's default size
        const.img_height = 224
        const.img_width = 224
        const.img_dimension = 3
        # metrics_name's indexes
        const.index_accuracy = 1

    # initialize random number generator
    np.random.seed(const.seed)
    random.seed(const.seed)

    # -------------------------------------------------------------------------
    # 1. load the data
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n1. load data")

    train_datagen = image.ImageDataGenerator()
    val_datagen = image.ImageDataGenerator()

    train_generator = train_datagen.flow_from_directory(
        directory=const.train_dir_path,
        target_size=(const.img_height, const.img_width),
        seed=const.seed,
        follow_links=True
    )
    val_generator = val_datagen.flow_from_directory(
        directory=const.val_dir_path,
        target_size=(const.img_height, const.img_width),
        seed=const.seed,
        follow_links=True
    )

    # train_generator and val_generator output: x (data) and y (label)
    # where x.shape == (32, 224, 224, 3) and y.shape == (32, 1056)

    const.logger.debug("TRAINING: number of classes: {}".format(train_generator.nb_class))
    const.logger.debug("TRAINING: number of images: {}".format(train_generator.nb_sample))
    for i in train_generator:
        x, y = i
        const.logger.debug("TRAINING: shape of x: {}, y: {}".format(x.shape, y.shape))
        break

    const.logger.debug("VALIDATION: number of classes: {}".format(val_generator.nb_class))
    const.logger.debug("VALIDATION: number of images: {}".format(val_generator.nb_sample))

    # -------------------------------------------------------------------------
    # 2. load the convolutional model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n2. load conv: {}".format(const.conv_model_type))

    model = {
        ConvType.ResNet50.value: __get_resnet50(include_top=False)
    }.get(const.conv_model_type)

    # -------------------------------------------------------------------------
    # 3. set the convolutional model to non-trainable
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n3. freeze conv")

    for layer in model.layers:
        layer.trainable=False

    # -------------------------------------------------------------------------
    # 4. extract features from the convolutional model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n4. extract features from conv")

    if const.train_features_path is not None:
        const.logger.info("TRAINING: load features")
        train_features = __load_features(const.train_features_path)
    else:
        const.logger.info("TRAINING: extract features")
        train_features, train_features_path = __extract_features(model, train_generator, "train")

    if const.val_features_path is not None:
        const.logger.info("VALIDATION: load features")
        val_features = __load_features(const.val_features_path)
    else:
        const.logger.info("VALIDATION: extract features")
        val_features, val_features_path = __extract_features(model, val_generator, "validation")

    # -------------------------------------------------------------------------
    # 5. build the classifier model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n5. build classifier")

    # think of features as inputs
    # model.output_shape = [1, nb_samples, nb_steps, nb_features (aka input_dimensions)]
    nb_features = model.output_shape[-1] # same as train_features.shape[-1]

    # think of classes as outputs
    nb_classes = train_generator.nb_class

    # taking the mean of the input and output size to get the hidden layer size
    nb_hidden_layers = int(round(np.mean([nb_features, nb_classes])))

    const.logger.info("{} -> [hidden layer {}] -> {}\n".format(nb_features, nb_hidden_layers, nb_classes))

    # use sequential instead of computational graph as our model
    top_model = Sequential()
    # add a non-linear function: recitified linear unit (relu)
    top_model.add(Dense(nb_hidden_layers, activation="relu", input_shape=train_features.shape[1:]))
    # dropout helps to protect the model from memorizing or overfitting the training data
    #top_model.add(Dropout(const.dropout_rate))
    # flatten 3D to 1D
    top_model.add(Flatten())
    # "softmax" ensures the output is a valid probability distribution
    # that is, the values are all non-negative and sum to 1
    # think of softmax as a multi-class sigmoid
    top_model.add(Dense(nb_classes, activation="softmax"))

    # -------------------------------------------------------------------------
    # 6. compile the classifier model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n6. compile classifier")

    top_model.compile(
        loss="categorical_crossentropy",
        optimizer="adam",
        metrics=["accuracy"]
    )

    # -------------------------------------------------------------------------
    # 7. create checkpoints to save classifier model's  weights
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n7. create checkpoints to save classifier's weights")

    callbacks_list = __create_checkpoints("classifier")

    # -------------------------------------------------------------------------
    # 8. train the classifier model using features
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n8. train classifier using features")

    top_model.fit_generator(
        generator=__create_generator_from_features(train_features, train_generator),
        samples_per_epoch=train_generator.batch_size,
        nb_epoch=const.nb_epoch,
        #callbacks_list=callbacks_list,
        verbose=2
    )

    # -------------------------------------------------------------------------
    # 9. evaluate the classifier model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n9. evaluate classifier")

    const.logger.info("extract labels for validation features")
    val_labels = np.zeros((val_features.shape[0], val_generator.nb_class))
    for i, class_index in zip(range(0, val_features.shape[0]), val_generator.classes):
        val_labels[i][class_index] = 1

    const.logger.info("evaluate classifier with validation features and labels")
    score = top_model.evaluate(
        x=val_features,
        y=val_labels,
        batch_size=val_generator.batch_size,
    )

    const.logger.info("{}: {}".format(
        top_model.metrics_names[const.index_accuracy],
        score[const.index_accuracy]
    ))

    # -------------------------------------------------------------------------
    # 10. set the convolutional model to trainable
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n10. unfreeze conv")

    for layer in model.layers:
        layer.trainable=True

    # -------------------------------------------------------------------------
    # 11. combine the convolutional model and classifier model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n11. add classifier on top of conv == combined")

    model = Model(input=model.input, output=top_model(model.output))

    # -------------------------------------------------------------------------
    # 12. compile the combined model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n12. compile combined")

    model.compile(
        loss="categorical_crossentropy",
        optimizer="adam",
        metrics=["accuracy"]
    )

    # -------------------------------------------------------------------------
    # 13. train the combined model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n13. train combined using data")

    model.fit_generator(
        generator=train_generator,
        samples_per_epoch=train_generator.batch_size,
        nb_epoch=const.nb_epoch,
        #callbacks=callbacks_list,
        verbose=2
    )

    # -------------------------------------------------------------------------
    # 14. evaluate the combined model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n14. evaluate combined")

    score = model.evaluate_generator(
        generator=val_generator,
        val_samples=val_generator.nb_sample
    )

    const.logger.info("{}: {}".format(
        model.metrics_names[1],
        score[1]
    ))

# =============================================================================
# Start here
# =============================================================================

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="cnn_retrainer.py",
        description="Generate a convolutional neural network retrained on car images",
        formatter_class=argparse.RawTextHelpFormatter
    )
    # arguments
    parser.add_argument("train_dir_path", default="train_dir_path", action="store", type=str,
        help="Specify the directory where training dataset lies.")
    parser.add_argument("val_dir_path", default="val_dir_path", action="store", type=str,
        help="Specify the directory where validation dataset lies.")
    # options
    parser.add_argument("-v", "--version", action="version",
        version="CNN Retrainer 1.0")
    parser.add_argument("--train_features_path", dest="train_features_path", action="store", type=str,
        default=None,
        help="Specify the file that contains the train features.")
    parser.add_argument("--val_features_path", dest="val_features_path", action="store", type=str,
        default=None,
        help="Specify the file that contains the validation features.")
    parser.add_argument("-w", dest="dataset_type", action="store", choices=["DGCarsDataset"], type=str,
        default="DGCarsDataset",
        help="Specify the datasets to use.")
    parser.add_argument("-c", dest="conv_model_type", action="store", choices=["ResNet50"], type=str,
        default="ResNet50",
        help="Specify the convolutional model to use.")
    parser.add_argument("-e", dest="nb_epoch", action="store", type=int,
        default=10,
        help="Specify epoch, the total number of iterations on the data.")
    parser.add_argument("-d", dest="dropout_rate", action="store", type=float,
        default=0.5,
        help="Specify dropout rate. Dropout helps model from not memorizing or overfitting the data.")
    parser.add_argument("-s", dest="seed", action="store", type=int,
        default=random.randint(1, 100),
        help="(OPTIONAL) SEED is used for random number generator.")
    main(parser.parse_args())<commit_msg>create checkpoint to save weights<commit_after>import argparse
import datetime
import enum
import logging
import numpy as np
import os
import random
import sys
import tempfile
import tensorflow as tf

from keras.applications.imagenet_utils import preprocess_input
from keras.applications.resnet50 import ResNet50
from keras.callbacks import ModelCheckpoint
from keras.models import Model, Sequential
from keras.layers import Dense, Dropout, Flatten, Input, Reshape
from keras.preprocessing import image

import pelops.const as const
from pelops.datasets.dgcars import DGCarsDataset
from pelops.utils import SetType, setup_custom_logger

# =============================================================================
# Allocate GPU memory so the program does not take over the entire GPU memory
# =============================================================================

def __get_session(gpu_fraction=0.3):
    '''Assume that you have 6GB of GPU memory and want to allocate ~2GB'''

    num_threads = os.environ.get('OMP_NUM_THREADS')
    const.logger.info("num_threads: {}".format(num_threads))
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)

    if num_threads:
        return tf.Session(config=tf.ConfigProto(
            gpu_options=gpu_options, intra_op_parallelism_threads=num_threads))
    else:
        return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

# =============================================================================
# Define convolutional model types
# =============================================================================

class ConvType(enum.Enum):
    ResNet50 = "ResNet50"

# =============================================================================
# Helper functions
# =============================================================================

def __get_resnet50(include_top):
    return ResNet50(
            include_top=include_top,
            weights="imagenet",
            input_tensor=Input(
                shape=(
                    const.img_height,
                    const.img_width,
                    const.img_dimension
                )
            )
    )

def __extract_features(model, generator, set_type):
    feature_dirpath = "./features/"
    const.logger.info("create a feature directory to store saved features: {}".format(feature_dirpath))
    if not os.path.exists(feature_dirpath):
        os.makedirs(feature_dirpath)

    const.logger.info("extract features from cnn")
    const.logger.debug("batch_size: {}".format(generator.batch_size))
    features = model.predict_generator(
        generator,
        generator.batch_size
    )

    time_now = datetime.datetime.now().strftime("%Y%m%d_%H_%M_%S")
    features_filepath = feature_dirpath + "{}_{}_{}_features_{}.npy".format(
        const.dataset_type,
        const.conv_model_type,
        set_type,
        time_now
    )
    const.logger.info("save features to {}".format(features_filepath))
    np.save(open(features_filepath, "wb"), features)

    return features, features_filepath

def __load_features(feature_filepath):
    return np.load(open(feature_filepath, "rb"))

def __create_checkpoints(which_model):
    checkpoint_dirpath = "./checkpoints/"
    const.logger.info("create a checkpoint directory to store saved checkpoints: {}".format(checkpoint_dirpath))
    if not os.path.exists(checkpoint_dirpath):
        os.makedirs(checkpoint_dirpath)

    checkpoint_filepath = \
        checkpoint_dirpath + \
        "{}_{}_best_checkpoint_".format(const.dataset_type, which_model) + \
        "{epoch:02d}_{acc:.4f}.npy"

    # TODO: it cannot monitor val_acc
    checkpoint = ModelCheckpoint(
        filepath=checkpoint_filepath,
        monitor="acc",
        save_best_only=True,
        mode="max",
        save_weights_only=False,
        verbose=2,
    )

    return [checkpoint]

def __create_generator_from_features(features, generator):

    nb_sample = generator.nb_sample
    nb_class = generator.nb_class
    batch_size = generator.batch_size

    # create labels for the features
    count = 0
    labels = np.zeros((batch_size, nb_class))
    for i, class_index in zip(range(0, batch_size), generator.classes):
        labels[i][class_index] = 1

    # create generator with features and labels
    while True:
        for i in range(int(nb_sample/batch_size)):
            x = features[i * batch_size: (i+1) * batch_size]
            y = labels[i * batch_size: (i+1) * batch_size]
            yield x, y

def __get_weights_filepath(which_model):
    weight_dirpath = "./weights/"
    const.logger.info("create a weights directory to store saved weights: {}".format(weight_dirpath))
    if not os.path.exists(weight_dirpath):
        os.makedirs(weight_dirpath)

    time_now = datetime.datetime.now().strftime("%Y%m%d_%H_%M_%S")
    weight_filepath = weight_dirpath + "{}_{}_weights_{}.npy".format(
        const.dataset_type,
        which_model,
        time_now
    )

    const.logger.info("save weights to {}".format(weight_filepath))
    return weight_filepath

# =============================================================================
# Main
# =============================================================================

def main(args):
    # setup logging
    const.logger = setup_custom_logger(__name__)

    # allocate gpu memory
    # TODO: causes memory error when run
    #session_number = __get_session()
    #const.logger.info("allocate GPU memory, session number: {}".format(session_number))

    # extract arguments from command line
    const.train_dir_path = args.train_dir_path
    const.val_dir_path = args.val_dir_path
    const.train_features_path = args.train_features_path
    const.val_features_path = args.val_features_path
    const.dataset_type = args.dataset_type
    const.conv_model_type = args.conv_model_type
    const.nb_epoch = args.nb_epoch
    const.dropout_rate = args.dropout_rate
    const.seed = args.seed

    # initialize constants
    if const.conv_model_type == ConvType.ResNet50.value:
        # ResNet50's default size
        const.img_height = 224
        const.img_width = 224
        const.img_dimension = 3
        # metrics_name's indexes
        const.index_accuracy = 1

    # initialize random number generator
    np.random.seed(const.seed)
    random.seed(const.seed)

    # -------------------------------------------------------------------------
    # 1. load the data
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n1. load data")

    train_datagen = image.ImageDataGenerator()
    val_datagen = image.ImageDataGenerator()

    train_generator = train_datagen.flow_from_directory(
        directory=const.train_dir_path,
        target_size=(const.img_height, const.img_width),
        seed=const.seed,
        follow_links=True
    )
    val_generator = val_datagen.flow_from_directory(
        directory=const.val_dir_path,
        target_size=(const.img_height, const.img_width),
        seed=const.seed,
        follow_links=True
    )

    # train_generator and val_generator output: x (data) and y (label)
    # where x.shape == (32, 224, 224, 3) and y.shape == (32, 1056)

    const.logger.debug("TRAINING: number of classes: {}".format(train_generator.nb_class))
    const.logger.debug("TRAINING: number of images: {}".format(train_generator.nb_sample))
    for i in train_generator:
        x, y = i
        const.logger.debug("TRAINING: shape of x: {}, y: {}".format(x.shape, y.shape))
        break

    const.logger.debug("VALIDATION: number of classes: {}".format(val_generator.nb_class))
    const.logger.debug("VALIDATION: number of images: {}".format(val_generator.nb_sample))

    # -------------------------------------------------------------------------
    # 2. load the convolutional model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n2. load conv: {}".format(const.conv_model_type))

    model = {
        ConvType.ResNet50.value: __get_resnet50(include_top=False)
    }.get(const.conv_model_type)

    # -------------------------------------------------------------------------
    # 3. set the convolutional model to non-trainable
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n3. freeze conv")

    for layer in model.layers:
        layer.trainable=False

    # -------------------------------------------------------------------------
    # 4. extract features from the convolutional model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n4. extract features from conv")

    if const.train_features_path is not None:
        const.logger.info("TRAINING: load features")
        train_features = __load_features(const.train_features_path)
    else:
        const.logger.info("TRAINING: extract features")
        train_features, train_features_path = __extract_features(model, train_generator, "train")

    if const.val_features_path is not None:
        const.logger.info("VALIDATION: load features")
        val_features = __load_features(const.val_features_path)
    else:
        const.logger.info("VALIDATION: extract features")
        val_features, val_features_path = __extract_features(model, val_generator, "validation")

    # -------------------------------------------------------------------------
    # 5. build the classifier model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n5. build classifier")

    # think of features as inputs
    # model.output_shape = [1, nb_samples, nb_steps, nb_features (aka input_dimensions)]
    nb_features = model.output_shape[-1] # same as train_features.shape[-1]

    # think of classes as outputs
    nb_classes = train_generator.nb_class

    # taking the mean of the input and output size to get the hidden layer size
    nb_hidden_layers = int(round(np.mean([nb_features, nb_classes])))

    const.logger.info("{} -> [hidden layer {}] -> {}\n".format(nb_features, nb_hidden_layers, nb_classes))

    # use sequential instead of computational graph as our model
    top_model = Sequential()
    # add a non-linear function: recitified linear unit (relu)
    top_model.add(Dense(nb_hidden_layers, activation="relu", input_shape=train_features.shape[1:]))
    # dropout helps to protect the model from memorizing or overfitting the training data
    #top_model.add(Dropout(const.dropout_rate))
    # flatten 3D to 1D
    top_model.add(Flatten())
    # "softmax" ensures the output is a valid probability distribution
    # that is, the values are all non-negative and sum to 1
    # think of softmax as a multi-class sigmoid
    top_model.add(Dense(nb_classes, activation="softmax"))

    # -------------------------------------------------------------------------
    # 6. compile the classifier model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n6. compile classifier")

    top_model.compile(
        loss="categorical_crossentropy",
        optimizer="adam",
        metrics=["accuracy"]
    )

    # -------------------------------------------------------------------------
    # 7. create checkpoints to save classifier model's  weights
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n7. create checkpoints to save classifier's weights")

    callbacks_list = __create_checkpoints("classifier")

    # -------------------------------------------------------------------------
    # 8. train the classifier model using features
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n8. train classifier using features")

    top_model.fit_generator(
        generator=__create_generator_from_features(train_features, train_generator),
        samples_per_epoch=train_generator.batch_size,
        nb_epoch=const.nb_epoch,
        callbacks=callbacks_list,
        verbose=2
    )

    top_model.save_weights(__get_weights_filepath("classifier"))

    # -------------------------------------------------------------------------
    # 9. evaluate the classifier model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n9. evaluate classifier")

    const.logger.info("extract labels for validation features")
    val_labels = np.zeros((val_features.shape[0], val_generator.nb_class))
    for i, class_index in zip(range(0, val_features.shape[0]), val_generator.classes):
        val_labels[i][class_index] = 1

    const.logger.info("evaluate classifier with validation features and labels")
    score = top_model.evaluate(
        x=val_features,
        y=val_labels,
        batch_size=val_generator.batch_size,
    )

    const.logger.info("{}: {}".format(
        top_model.metrics_names[const.index_accuracy],
        score[const.index_accuracy]
    ))

    # -------------------------------------------------------------------------
    # 10. set the convolutional model to trainable
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n10. unfreeze conv")

    for layer in model.layers:
        layer.trainable=True

    # -------------------------------------------------------------------------
    # 11. combine the convolutional model and classifier model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n11. add classifier on top of conv == combined")

    model = Model(input=model.input, output=top_model(model.output))

    # -------------------------------------------------------------------------
    # 12. compile the combined model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n12. compile combined")

    model.compile(
        loss="categorical_crossentropy",
        optimizer="adam",
        metrics=["accuracy"]
    )

    # -------------------------------------------------------------------------
    # 13. create checkpoints to save classifier model's  weights
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n7. create checkpoints to save classifier's weights")

    callbacks_list = __create_checkpoints("combined")

    # -------------------------------------------------------------------------
    # 14. train the combined model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n13. train combined using data")

    model.fit_generator(
        generator=train_generator,
        samples_per_epoch=train_generator.batch_size,
        nb_epoch=const.nb_epoch,
        callbacks=callbacks_list,
        verbose=2
    )

    model.save_weights(__get_weights_filepath("combined"))

    # -------------------------------------------------------------------------
    # 15. evaluate the combined model
    # -------------------------------------------------------------------------

    const.logger.info("=" * 80 + "\n14. evaluate combined")

    score = model.evaluate_generator(
        generator=val_generator,
        val_samples=val_generator.nb_sample
    )

    const.logger.info("{}: {}".format(
        model.metrics_names[1],
        score[1]
    ))

# =============================================================================
# Start here
# =============================================================================

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="cnn_retrainer.py",
        description="Generate a convolutional neural network retrained on car images",
        formatter_class=argparse.RawTextHelpFormatter
    )
    # arguments
    parser.add_argument("train_dir_path", default="train_dir_path", action="store", type=str,
        help="Specify the directory where training dataset lies.")
    parser.add_argument("val_dir_path", default="val_dir_path", action="store", type=str,
        help="Specify the directory where validation dataset lies.")
    # options
    parser.add_argument("-v", "--version", action="version",
        version="CNN Retrainer 1.0")
    parser.add_argument("--train_features_path", dest="train_features_path", action="store", type=str,
        default=None,
        help="Specify the file that contains the train features.")
    parser.add_argument("--val_features_path", dest="val_features_path", action="store", type=str,
        default=None,
        help="Specify the file that contains the validation features.")
    parser.add_argument("-w", dest="dataset_type", action="store", choices=["DGCarsDataset"], type=str,
        default="DGCarsDataset",
        help="Specify the datasets to use.")
    parser.add_argument("-c", dest="conv_model_type", action="store", choices=["ResNet50"], type=str,
        default="ResNet50",
        help="Specify the convolutional model to use.")
    parser.add_argument("-e", dest="nb_epoch", action="store", type=int,
        default=10,
        help="Specify epoch, the total number of iterations on the data.")
    parser.add_argument("-d", dest="dropout_rate", action="store", type=float,
        default=0.5,
        help="Specify dropout rate. Dropout helps model from not memorizing or overfitting the data.")
    parser.add_argument("-s", dest="seed", action="store", type=int,
        default=random.randint(1, 100),
        help="(OPTIONAL) SEED is used for random number generator.")
    main(parser.parse_args())<|endoftext|>


<commit_before>import kurve
import math
from nc.nc import *

def make_smaller( k, startx = None, starty = None, finishx = None, finishy = None ):
    if startx != None or starty != None:
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, 0)

        if startx == None: startx = sx
        if starty == None: starty = sy

        kurve.change_start(k, startx, starty)

    if finishx != None or finishy != None:
        num_spans = kurve.num_spans(k)
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, num_spans - 1)

        if finishx == None: finishx = ex
        if finishy == None: finishy = ey

        kurve.change_end(k, finishx, finishy)

def split_for_tags( k, radius, start_depth, depth, tag ):
    num_tags, tag_width, tag_angle, tag_at_start = tag
    tag_height = tag_width/2 * math.tan(tag_angle)
    perim = kurve.perim(k)
    sub_perim = perim / num_tags
    half_flat_top = radius
    if tag_height > (start_depth - depth):
        # cut the top off the tag
        chop_off_height = tag_height - (start_depth - depth)
        half_flat_top += chop_off_height / math.tan(tag_angle)

    for i in range(0, num_tags):
        d = sub_perim * i
        if tag_at_start == False: d += sub_perim / 2

        d0 = d - half_flat_top
        while d0 < 0: d0 += perim
        while d0 > perim: d0 -= perim
        px, py = kurve.perim_to_point(k, d0)
        kurve.kbreak(k, px, py)
        d1 = d + half_flat_top
        while d1 < 0: d1 += perim
        while d1 > perim: d1 -= perim
        px, py = kurve.perim_to_point(k, d1)
        kurve.kbreak(k, px, py)

        if tag_width + 2 * radius > sub_perim:
            px, py = kurve.perim_to_point(k, d + sub_perim/2)
            kurve.kbreak(k, px, py)
        else:
            d0 = d - tag_width/2 - radius
            while d0 < 0: d0 += perim
            while d0 > perim: d0 -= perim
            px, py = kurve.perim_to_point(k, d0)
            kurve.kbreak(k, px, py)
            d1 = d + tag_width/2 + radius
            while d1 < 0: d1 += perim
            while d1 > perim: d1 -= perim
            px, py = kurve.perim_to_point(k, d1)
            kurve.kbreak(k, px, py)

def get_tag_z_for_span(current_perim, k, radius, start_depth, depth, tag):
    num_tags, tag_width, tag_angle, tag_at_start = tag
    tag_height = tag_width/2 * math.tan(tag_angle)
    perim = kurve.perim(k)
    sub_perim = perim / num_tags
    max_z = depth
    for i in range(0, num_tags + 1):
        d = sub_perim * i
        if (tag_at_start == False) and (i != num_tags + 1): d += sub_perim / 2
        dist_from_d = math.fabs(current_perim - d) - radius
        if dist_from_d < 0: dist_from_d = 0
        z = (depth + tag_height) - (dist_from_d / ( tag_width / 2 )) * tag_height
        if z > max_z: max_z = z

    if max_z > start_depth:
        max_z = start_depth

    return max_z

def add_roll_on(k, direction, roll_radius, offset_extra, roll_on):
    if direction == "on": return
    if roll_on == None: return
    num_spans = kurve.num_spans(k)
    if num_spans == 0: return

    if roll_on == 'auto':
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, 0)
        vx, vy = kurve.get_span_dir(k, 0, 0) # get start direction
        if direction == 'right':
            off_vx = vy
            off_vy = -vx
        else:
            off_vx = -vy
            off_vy = vx

        rollstartx = sx + off_vx * roll_radius - vx * roll_radius
        rollstarty = sy + off_vy * roll_radius - vy * roll_radius
    else:
        rollstartx, rollstarty =  roll_on
    copy_kurve = kurve.new()
    sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, 0)
    if sx == rollstartx and sy == rollstarty: return
    vx, vy = kurve.get_span_dir(k, 0, 0) # get start direction
    rcx, rcy, rdir = kurve.tangential_arc(sx, sy, -vx, -vy, rollstartx, rollstarty)
    rdir = -rdir # because the tangential_arc was used in reverse

    kurve.add_point(copy_kurve, 0, rollstartx, rollstarty, 0, 0)
    kurve.add_point(copy_kurve, rdir, sx, sy, rcx, rcy)

    # add all the other spans

    for span in range(0, num_spans):
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, span)
        kurve.add_point(copy_kurve, sp, ex, ey, cx, cy)

    # copy back to k
    kurve.copy(copy_kurve, k)

    # delete copy
    kurve.delete(copy_kurve)

def add_roll_off(k, direction, roll_radius, offset_extra, roll_off):
    if direction == "on": return
    if roll_off == None: return
    num_spans = kurve.num_spans(k)
    if num_spans == 0: return

    if roll_off == 'auto':
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, num_spans - 1)
        vx, vy = kurve.get_span_dir(k, num_spans - 1, 1) # get end direction
        if direction == 'right':
            off_vx = vy
            off_vy = -vx
        else:
            off_vx = -vy
            off_vy = vx

        rollendx = ex + off_vx * roll_radius + vx * roll_radius;
        rollendy = ey + off_vy * roll_radius + vy * roll_radius;
    else:
        rollendx, rollendy =  roll_off

    sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, num_spans - 1)
    if ex == rollendx and ey == rollendy: return
    vx, vy = kurve.get_span_dir(k, num_spans - 1, 1) # get end direction
    rcx, rcy, rdir = kurve.tangential_arc(ex, ey, vx, vy, rollendx, rollendy)

    kurve.add_point(k, rdir, rollendx, rollendy, rcx, rcy)

# profile command,
# direction should be 'left' or 'right' or 'on'
def profile(k, direction = "on", radius = 1.0, offset_extra = 0.0, roll_radius = 2.0, roll_on = None, roll_off = None, tag = None, rapid_down_to_height = None, start_depth = None, step_down = None, final_depth = None):
    if kurve.exists(k) == False:
        raise "kurve doesn't exist, number %d" % (k)

    offset_k = k

    if direction != "on":
        if direction != "left" and direction != "right":
            raise "direction must be left or right", direction

        # get tool diameter
        offset = radius + offset_extra
        if use_CRC():
            add_CRC_start_line(offset_k, radius)
            start_CRC(direction == "left", radius)
        else:
            if direction == "right":
                offset = -offset
            offset_k = kurve.new()
            offset_success = kurve.offset(k, offset_k, offset)
            if offset_success == False:
                raise "couldn't offset kurve %d" % (k)

    if kurve.num_spans(offset_k) == 0:
        raise "sketch has no spans!"

    current_perim = 0.0

    # add the roll on and roll off moves to the kurve
    add_roll_on(offset_k, direction, roll_radius, offset_extra, roll_on)
    add_roll_off(offset_k, direction, roll_radius, offset_extra, roll_on)

    # do multiple depths
    total_to_cut = start_depth - final_depth;
    num_step_downs = int(float(total_to_cut) / math.fabs(step_down) + 0.999999)

    # tag
    if tag != None:
        num_tags, tag_width, tag_angle, tag_at_start = tag
        if tag_width < 0.00001:
            tag = None
        else:
            tag_height = float(tag_width)/2 * math.tan(tag_angle)
            tag_depth = final_depth + tag_height
            if tag_depth > start_depth: tag_depth = start_depth
            copy_of_offset_k = kurve.new()
            kurve.copy(offset_k, copy_of_offset_k)

    incremental_rapid_height = rapid_down_to_height - start_depth
    prev_depth = start_depth
    for step in range(0, num_step_downs):
        depth_of_cut = ( start_depth - final_depth ) * ( step + 1 ) / num_step_downs
        depth = start_depth - depth_of_cut
        mat_depth = prev_depth
        if tag != None and tag_depth > mat_depth: mat_depth = tag_depth

        if tag != None:
            tag_height = tag_width/2 * math.tan(tag_angle)
            perim = kurve.perim(offset_k)
            sub_perim = perim / num_tags
            lowest_z = (depth + tag_height) - ((sub_perim / 2) / ( tag_width / 2 )) * tag_height
            if lowest_z > start_depth:
                return # the whole layer is tag
            split_for_tags(offset_k, radius, start_depth, depth, tag)

        for span in range(0, kurve.num_spans(offset_k)):
            sp, sx, sy, ex, ey, cx, cy = kurve.get_span(offset_k, span)
            if span == 0:#first span
                # rapid across to it
                rapid(sx, sy)

                # rapid down to just above the material, which might be at the top of the tag
                rapid(z = mat_depth + incremental_rapid_height)

                # feed down to depth
                mat_depth = depth
                if tag != None and tag_depth > mat_depth: mat_depth = tag_depth
                feed(z = mat_depth)

            # height for tags
            current_perim += kurve.get_span_length(offset_k, span)
            ez = None
            if tag != None:
                ez = get_tag_z_for_span(current_perim, offset_k, radius, start_depth, depth, tag)

            if sp == 0:#line
                feed(ex, ey, ez)
            else:
                cx = cx - sx # make relative to the start position
                cy = cy - sy
                if sp == 1:# anti-clockwise arc
                    arc_ccw(ex, ey, ez, i = cx, j = cy)
                else:
                    arc_cw(ex, ey, ez, i = cx, j = cy)
        if tag != None:
            kurve.copy(copy_of_offset_k, offset_k)

    if offset_k != k:
        kurve.delete(offset_k)

    if tag != None:
        kurve.delete(copy_of_offset_k)

    if use_CRC():
        end_CRC()

def roll_on_point(k, direction, radius, roll_radius):
    x = float(0)
    y = float(0)

    offset = radius
    if direction == "right":
        offset = -offset
    offset_k = kurve.new()
    offset_success = kurve.offset(k, offset_k, offset)
    if offset_success == False:
        raise "couldn't offset kurve %d" % (k)

    if kurve.num_spans(offset_k) > 0:
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(offset_k, 0)
        vx, vy = kurve.get_span_dir(offset_k, 0, 0) # get start direction
        off_vx = -vy
        off_vy = vx
        if direction == 'right':
            off_vx = -off_vx
            off_vy = -off_vy
        x = sx + off_vx * roll_radius - vx * roll_radius
        y = sy + off_vy * roll_radius - vy * roll_radius

    return x, y

def roll_off_point(k, direction, radius, roll_radius):
    x = float(0)
    y = float(0)

    offset = radius
    if direction == "right":
        offset = -offset
    offset_k = kurve.new()
    offset_success = kurve.offset(k, offset_k, offset)
    if offset_success == False:
        raise "couldn't offset kurve %d" % (k)

    n = kurve.num_spans(offset_k)

    if n > 0:
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(offset_k, n - 1)
        vx, vy = kurve.get_span_dir(offset_k, n - 1, 1) # get end direction
        off_vx = -vy
        off_vy = vx
        if direction == 'right':
            off_vx = -off_vx
            off_vy = -off_vy
        x = ex + off_vx * roll_radius + vx * roll_radius
        y = ey + off_vy * roll_radius + vy * roll_radius

    return x, y
<commit_msg>I changed order of rapids from rapid(sx,sy)>rapid(z = mat_depth + incremental_rapid_height) to rapid(z = mat_depth + incremental_rapid_height)>rapid(sx,sy). The tool was rapiding across and through part if a profile was open. I'm not sure how this will affect tags because I haven't tried them yet<commit_after>import kurve
import math
from nc.nc import *

def make_smaller( k, startx = None, starty = None, finishx = None, finishy = None ):
    if startx != None or starty != None:
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, 0)

        if startx == None: startx = sx
        if starty == None: starty = sy

        kurve.change_start(k, startx, starty)

    if finishx != None or finishy != None:
        num_spans = kurve.num_spans(k)
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, num_spans - 1)

        if finishx == None: finishx = ex
        if finishy == None: finishy = ey

        kurve.change_end(k, finishx, finishy)

def split_for_tags( k, radius, start_depth, depth, tag ):
    num_tags, tag_width, tag_angle, tag_at_start = tag
    tag_height = tag_width/2 * math.tan(tag_angle)
    perim = kurve.perim(k)
    sub_perim = perim / num_tags
    half_flat_top = radius
    if tag_height > (start_depth - depth):
        # cut the top off the tag
        chop_off_height = tag_height - (start_depth - depth)
        half_flat_top += chop_off_height / math.tan(tag_angle)

    for i in range(0, num_tags):
        d = sub_perim * i
        if tag_at_start == False: d += sub_perim / 2

        d0 = d - half_flat_top
        while d0 < 0: d0 += perim
        while d0 > perim: d0 -= perim
        px, py = kurve.perim_to_point(k, d0)
        kurve.kbreak(k, px, py)
        d1 = d + half_flat_top
        while d1 < 0: d1 += perim
        while d1 > perim: d1 -= perim
        px, py = kurve.perim_to_point(k, d1)
        kurve.kbreak(k, px, py)

        if tag_width + 2 * radius > sub_perim:
            px, py = kurve.perim_to_point(k, d + sub_perim/2)
            kurve.kbreak(k, px, py)
        else:
            d0 = d - tag_width/2 - radius
            while d0 < 0: d0 += perim
            while d0 > perim: d0 -= perim
            px, py = kurve.perim_to_point(k, d0)
            kurve.kbreak(k, px, py)
            d1 = d + tag_width/2 + radius
            while d1 < 0: d1 += perim
            while d1 > perim: d1 -= perim
            px, py = kurve.perim_to_point(k, d1)
            kurve.kbreak(k, px, py)

def get_tag_z_for_span(current_perim, k, radius, start_depth, depth, tag):
    num_tags, tag_width, tag_angle, tag_at_start = tag
    tag_height = tag_width/2 * math.tan(tag_angle)
    perim = kurve.perim(k)
    sub_perim = perim / num_tags
    max_z = depth
    for i in range(0, num_tags + 1):
        d = sub_perim * i
        if (tag_at_start == False) and (i != num_tags + 1): d += sub_perim / 2
        dist_from_d = math.fabs(current_perim - d) - radius
        if dist_from_d < 0: dist_from_d = 0
        z = (depth + tag_height) - (dist_from_d / ( tag_width / 2 )) * tag_height
        if z > max_z: max_z = z

    if max_z > start_depth:
        max_z = start_depth

    return max_z

def add_roll_on(k, direction, roll_radius, offset_extra, roll_on):
    if direction == "on": return
    if roll_on == None: return
    num_spans = kurve.num_spans(k)
    if num_spans == 0: return

    if roll_on == 'auto':
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, 0)
        vx, vy = kurve.get_span_dir(k, 0, 0) # get start direction
        if direction == 'right':
            off_vx = vy
            off_vy = -vx
        else:
            off_vx = -vy
            off_vy = vx

        rollstartx = sx + off_vx * roll_radius - vx * roll_radius
        rollstarty = sy + off_vy * roll_radius - vy * roll_radius
    else:
        rollstartx, rollstarty =  roll_on
    copy_kurve = kurve.new()
    sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, 0)
    if sx == rollstartx and sy == rollstarty: return
    vx, vy = kurve.get_span_dir(k, 0, 0) # get start direction
    rcx, rcy, rdir = kurve.tangential_arc(sx, sy, -vx, -vy, rollstartx, rollstarty)
    rdir = -rdir # because the tangential_arc was used in reverse

    kurve.add_point(copy_kurve, 0, rollstartx, rollstarty, 0, 0)
    kurve.add_point(copy_kurve, rdir, sx, sy, rcx, rcy)

    # add all the other spans

    for span in range(0, num_spans):
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, span)
        kurve.add_point(copy_kurve, sp, ex, ey, cx, cy)

    # copy back to k
    kurve.copy(copy_kurve, k)

    # delete copy
    kurve.delete(copy_kurve)

def add_roll_off(k, direction, roll_radius, offset_extra, roll_off):
    if direction == "on": return
    if roll_off == None: return
    num_spans = kurve.num_spans(k)
    if num_spans == 0: return

    if roll_off == 'auto':
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, num_spans - 1)
        vx, vy = kurve.get_span_dir(k, num_spans - 1, 1) # get end direction
        if direction == 'right':
            off_vx = vy
            off_vy = -vx
        else:
            off_vx = -vy
            off_vy = vx

        rollendx = ex + off_vx * roll_radius + vx * roll_radius;
        rollendy = ey + off_vy * roll_radius + vy * roll_radius;
    else:
        rollendx, rollendy =  roll_off

    sp, sx, sy, ex, ey, cx, cy = kurve.get_span(k, num_spans - 1)
    if ex == rollendx and ey == rollendy: return
    vx, vy = kurve.get_span_dir(k, num_spans - 1, 1) # get end direction
    rcx, rcy, rdir = kurve.tangential_arc(ex, ey, vx, vy, rollendx, rollendy)

    kurve.add_point(k, rdir, rollendx, rollendy, rcx, rcy)

# profile command,
# direction should be 'left' or 'right' or 'on'
def profile(k, direction = "on", radius = 1.0, offset_extra = 0.0, roll_radius = 2.0, roll_on = None, roll_off = None, tag = None, rapid_down_to_height = None, start_depth = None, step_down = None, final_depth = None):
    if kurve.exists(k) == False:
        raise "kurve doesn't exist, number %d" % (k)

    offset_k = k

    if direction != "on":
        if direction != "left" and direction != "right":
            raise "direction must be left or right", direction

        # get tool diameter
        offset = radius + offset_extra
        if use_CRC():
            add_CRC_start_line(offset_k, radius)
            start_CRC(direction == "left", radius)
        else:
            if direction == "right":
                offset = -offset
            offset_k = kurve.new()
            offset_success = kurve.offset(k, offset_k, offset)
            if offset_success == False:
                raise "couldn't offset kurve %d" % (k)

    if kurve.num_spans(offset_k) == 0:
        raise "sketch has no spans!"

    current_perim = 0.0

    # add the roll on and roll off moves to the kurve
    add_roll_on(offset_k, direction, roll_radius, offset_extra, roll_on)
    add_roll_off(offset_k, direction, roll_radius, offset_extra, roll_on)

    # do multiple depths
    total_to_cut = start_depth - final_depth;
    num_step_downs = int(float(total_to_cut) / math.fabs(step_down) + 0.999999)

    # tag
    if tag != None:
        num_tags, tag_width, tag_angle, tag_at_start = tag
        if tag_width < 0.00001:
            tag = None
        else:
            tag_height = float(tag_width)/2 * math.tan(tag_angle)
            tag_depth = final_depth + tag_height
            if tag_depth > start_depth: tag_depth = start_depth
            copy_of_offset_k = kurve.new()
            kurve.copy(offset_k, copy_of_offset_k)

    incremental_rapid_height = rapid_down_to_height - start_depth
    prev_depth = start_depth
    for step in range(0, num_step_downs):
        depth_of_cut = ( start_depth - final_depth ) * ( step + 1 ) / num_step_downs
        depth = start_depth - depth_of_cut
        mat_depth = prev_depth
        if tag != None and tag_depth > mat_depth: mat_depth = tag_depth

        if tag != None:
            tag_height = tag_width/2 * math.tan(tag_angle)
            perim = kurve.perim(offset_k)
            sub_perim = perim / num_tags
            lowest_z = (depth + tag_height) - ((sub_perim / 2) / ( tag_width / 2 )) * tag_height
            if lowest_z > start_depth:
                return # the whole layer is tag
            split_for_tags(offset_k, radius, start_depth, depth, tag)

        for span in range(0, kurve.num_spans(offset_k)):
            sp, sx, sy, ex, ey, cx, cy = kurve.get_span(offset_k, span)
            if span == 0:#first span
                #I changed order of rapids from rapid(sx,sy)>rapid(z = mat_depth + incremental_rapid_height)
                #to rapid(z = mat_depth + incremental_rapid_height)>rapid(sx,sy)
                #because tool was rapiding across and through part if profile was open
                #not sure how this will affect tags because I haven't tried them yet -Dan Falck-

                # rapid down to just above the material, which might be at the top of the tag
                rapid(z = mat_depth + incremental_rapid_height)
                # rapid across to it
                rapid(sx, sy)

                # feed down to depth
                mat_depth = depth
                if tag != None and tag_depth > mat_depth: mat_depth = tag_depth
                feed(z = mat_depth)

            # height for tags
            current_perim += kurve.get_span_length(offset_k, span)
            ez = None
            if tag != None:
                ez = get_tag_z_for_span(current_perim, offset_k, radius, start_depth, depth, tag)

            if sp == 0:#line
                feed(ex, ey, ez)
            else:
                cx = cx - sx # make relative to the start position
                cy = cy - sy
                if sp == 1:# anti-clockwise arc
                    arc_ccw(ex, ey, ez, i = cx, j = cy)
                else:
                    arc_cw(ex, ey, ez, i = cx, j = cy)
        if tag != None:
            kurve.copy(copy_of_offset_k, offset_k)

    if offset_k != k:
        kurve.delete(offset_k)

    if tag != None:
        kurve.delete(copy_of_offset_k)

    if use_CRC():
        end_CRC()

def roll_on_point(k, direction, radius, roll_radius):
    x = float(0)
    y = float(0)

    offset = radius
    if direction == "right":
        offset = -offset
    offset_k = kurve.new()
    offset_success = kurve.offset(k, offset_k, offset)
    if offset_success == False:
        raise "couldn't offset kurve %d" % (k)

    if kurve.num_spans(offset_k) > 0:
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(offset_k, 0)
        vx, vy = kurve.get_span_dir(offset_k, 0, 0) # get start direction
        off_vx = -vy
        off_vy = vx
        if direction == 'right':
            off_vx = -off_vx
            off_vy = -off_vy
        x = sx + off_vx * roll_radius - vx * roll_radius
        y = sy + off_vy * roll_radius - vy * roll_radius

    return x, y

def roll_off_point(k, direction, radius, roll_radius):
    x = float(0)
    y = float(0)

    offset = radius
    if direction == "right":
        offset = -offset
    offset_k = kurve.new()
    offset_success = kurve.offset(k, offset_k, offset)
    if offset_success == False:
        raise "couldn't offset kurve %d" % (k)

    n = kurve.num_spans(offset_k)

    if n > 0:
        sp, sx, sy, ex, ey, cx, cy = kurve.get_span(offset_k, n - 1)
        vx, vy = kurve.get_span_dir(offset_k, n - 1, 1) # get end direction
        off_vx = -vy
        off_vy = vx
        if direction == 'right':
            off_vx = -off_vx
            off_vy = -off_vy
        x = ex + off_vx * roll_radius + vx * roll_radius
        y = ey + off_vy * roll_radius + vy * roll_radius

    return x, y
<|endoftext|>