<commit_before>import logging
from collections import defaultdict
from operator import or_

import celery
from django.shortcuts import get_object_or_404

from rest_framework import (
    generics,
    parsers,
)
from rest_framework.response import Response

from core.exceptions import (
    ResourceConflict,
    BadRequest,
)

from volumes.api import VolumeListAPIView
from volumes.models import Volume
from blueprints.models import Blueprint

from . import tasks, models, serializers

logger = logging.getLogger(__name__)

class StackListAPIView(generics.ListCreateAPIView):
    '''
    TODO: Add docstring
    '''
    model = models.Stack
    serializer_class = serializers.StackSerializer
    parser_classes = (parsers.JSONParser,)

    def get_queryset(self):
        return self.request.user.stacks.all()

    def create(self, request, *args, **kwargs):
        '''
        Overriding create method to build roles and metadata objects for this Stack
        as well as generating the salt-cloud map that will be used to launch
        machines
        '''
        # make sure the user has a public key or they won't be able to SSH
        # later
        if not request.user.settings.public_key:
            raise BadRequest('You have not added a public key to your user '
                             'profile and will not be able to SSH in to any '
                             'machines. Please update your user profile '
                             'before continuing.')

        # Validate data
        errors = {}
        blueprint_id = request.DATA.pop('blueprint', '')
        title = request.DATA.get('title', '')
        description = request.DATA.get('description', '')
        properties = request.DATA.get('properties', {})

        # check for required blueprint
        if not blueprint_id:
            errors.setdefault('blueprint', []).append('This field is required.')
        else:
            try:
                blueprint = Blueprint.objects.get(pk=blueprint_id,
                                                  owner=request.user)
            except Blueprint.DoesNotExist:
                errors.setdefault('blueprint', []).append(
                    'Blueprint with id {0} does not exist.'.format(blueprint_id)
                )
            except ValueError:
                errors.setdefault('blueprint', []).append(
                    'This field must be an integer ID of an existing blueprint.'
                )

        if not title:
            errors.setdefault('title', []).append('This field is required.')

        if not description:
            errors.setdefault('description', []).append('This field is required.')

        # check for duplicates
        if models.Stack.objects.filter(owner=self.request.user, title=title).count():
            errors.setdefault('title', []).append(
                'A Stack with this title already exists in your account.'
            )

        if not isinstance(properties, dict):
            errors.setdefault('properties', []).append(
                'This field must be a JSON object.'
            )
        else:
            # user properties are not allowed to provide a __stackdio__ key
            if '__stackdio__' in properties:
                errors.setdefault('properties', []).append(
                    'The __stackdio__ key is reserved for system use.'
                )

        if errors:
            raise BadRequest(errors)

        # create the stack and related objects
        try:
            logger.debug(request.DATA)
            stack = models.Stack.objects.create_stack(request.user, blueprint, **request.DATA)
            raise Exception('foooooo')
        except Exception, e:
            logger.exception(e)
            raise BadRequest(str(e))

        # set some defaults
        launch_stack = request.DATA.get('auto_launch', True)
        provision_stack = request.DATA.get('auto_provision', True)

        if launch_stack:
            # Queue up stack creation and provisioning using Celery
            task_list = [
                tasks.launch_hosts.si(stack.id),
                tasks.update_metadata.si(stack.id),
                tasks.tag_infrastructure.si(stack.id),
                tasks.register_dns.si(stack.id),
                tasks.ping.si(stack.id),
                tasks.sync_all.si(stack.id)
            ]

            # provisioning is optional (mainly useful for getting machines
            # up so you can play with salt states)
            if provision_stack:
                task_list.append(tasks.highstate.si(stack.id))
                task_list.append(tasks.orchestrate.si(stack.id))

            # always finish
            task_list.append(tasks.finish_stack.si(stack.id))

            logger.debug(task_list)
            task_chain = reduce(or_, task_list)

            # execute the chain
            task_chain(link_error=tasks.handle_error.s(stack.id))

        # return serialized stack object
        serializer = serializers.StackSerializer(stack, context={
            'request': request,
        })
        return Response(serializer.data)

class StackDetailAPIView(generics.RetrieveUpdateDestroyAPIView):
    model = models.Stack
    serializer_class = serializers.StackSerializer
    parser_classes = (parsers.JSONParser,)

    def get_object(self):
        return get_object_or_404(models.Stack, id=self.kwargs.get('pk'),
                                 owner=self.request.user)

    def destroy(self, request, *args, **kwargs):
        '''
        Overriding the delete method to make sure the stack
        is taken offline before being deleted
        '''
        # Update the status
        stack = self.get_object()
        msg = 'Stack will be removed upon successful termination ' \
              'of all machines'
        stack.set_status(models.Stack.DESTROYING, msg)

        # Queue up stack destroy tasks
        task_chain = (
            tasks.register_volume_delete.si(stack.id) |
            tasks.unregister_dns.si(stack.id) |
            tasks.destroy_hosts.si(stack.id)
        )

        # execute the chain
        task_chain()

        # Return the stack while its deleting
        serializer = self.get_serializer(stack)
        return Response(serializer.data)

    def put(self, request, *args, **kwargs):
        '''
        PUT request on a stack allows RPC-like actions to be called to
        interact with the stack. Request data is JSON, and must provide
        an `action` parameter. Actions may require additional arguments
        which may be provided via the `args` parameter.

        Valid actions: stop, start, restart, terminate
        '''

        stack = self.get_object()
        hosts = stack.get_hosts()
        host_count = hosts.count()
        action = request.DATA.get('action', None)
        args = request.DATA.get('args', [])

        if not action:
            raise BadRequest('action is a required parameter.')

        driver = stack.get_driver()
        available_actions = driver.get_available_actions()

        if action not in available_actions:
            raise BadRequest('action is not alowed. Only the following '
                             'actions are allowed: {0}'.format(', '.join(available_actions)))

        # In case of a launch action, the stack must not have any available
        # hosts (ie, the stack must have already been terminated.)
        if action == driver.ACTION_LAUNCH and host_count > 0:
            raise BadRequest('Launching a stack is only available when '
                             'the stack is in a terminated state. This '
                             'stack has %d hosts available.' % host_count)

        # All other actions require hosts to be available
        elif action != driver.ACTION_LAUNCH and host_count == 0:
            raise BadRequest('The submitted action requires the stack to have '
                             'available hosts. Perhaps you meant to run the '
                             'launch action instead.')

        # check the action against current states (e.g., starting can't happen
        # unless the hosts are in the stopped state.)
        # XXX: Assuming that host metadata is accurate here
        for host in hosts:
            if action == driver.ACTION_START and \
               host.state != driver.STATE_STOPPED:
                raise BadRequest('Start action requires all hosts to be in '
                                 'the stopped state first. At least one host '
                                 'is reporting an invalid '
                                 'state: %s' % host.state)
            if action == driver.ACTION_STOP and \
               host.state != driver.STATE_RUNNING:
                raise BadRequest('Stop action requires all hosts to be in '
                                 'the running state first. At least one host '
                                 'is reporting an invalid '
                                 'state: %s' % host.state)
            if action == driver.ACTION_TERMINATE and \
               host.state not in (driver.STATE_RUNNING, driver.STATE_STOPPED):
                raise BadRequest('Terminate action requires all hosts to be '
                                 'in the either the running or stopped state '
                                 'first. At least one host is reporting an '
                                 'invalid state: %s' % host.state)
            if action == driver.ACTION_PROVISION and \
               host.state not in (driver.STATE_RUNNING,):
                raise BadRequest('Provision action requires all hosts to be '
                                 'in the running state first. At least one '
                                 'host is reporting an '
                                 'invalid state: %s' % host.state)

        # Kick off the celery task for the given action
        stack.set_status(models.Stack.EXECUTING_ACTION,
                         'Stack is executing action \'{0}\''.format(action))

        # Keep track of the tasks we need to run for this execution
        task_list = []

        # FIXME: not generic
        if action in (driver.ACTION_STOP, driver.ACTION_TERMINATE):
            # Unregister DNS when executing the above actions
            task_list.append(tasks.unregister_dns.si(stack.id))

        # Launch is slightly different than other actions
        if action == driver.ACTION_LAUNCH:
            task_list.append(tasks.launch_hosts.si(stack.id))

        # Terminate should leverage salt-cloud or salt gets confused about
        # the state of things
        elif action == driver.ACTION_TERMINATE:
            task_list.append(tasks.destroy_hosts.si(stack.id, delete_stack=False))

        # Execute other actions
        else:
            task_list.append(tasks.execute_action.si(stack.id, action, *args))

        # Update the metadata after the action has been executed
        task_list.append(tasks.update_metadata.si(stack.id))

        # Launching requires us to tag the newly available infrastructure
        if action in (driver.ACTION_LAUNCH,):
            tasks.tag_infrastructure.si(stack.id)

        # These actions require new DNS and provisioning
        if action in (driver.ACTION_START, driver.ACTION_LAUNCH):
            # Register DNS when executing the above actions
            task_list.append(tasks.register_dns.si(stack.id))
            # Also reprovision just in case?
            # TODO: do we need this and are there cases where special things
            # need to happen after a host is restarted?
            task_list.append(tasks.ping.si(stack.id))
            task_list.append(tasks.sync_all.si(stack.id))
            task_list.append(tasks.highstate.si(stack.id))
            task_list.append(tasks.orchestrate.si(stack.id))

        task_list.append(tasks.finish_stack.si(stack.id))

        # chain together our tasks using the bitwise or operator
        task_chain = reduce(or_, task_list)

        # Update all host states
        hosts.update(state='actioning')

        # execute the chain
        task_chain()

        serializer = self.get_serializer(stack)
        return Response(serializer.data)

class StackHistoryList(generics.ListAPIView):
    model = models.StackHistory
    serializer_class = serializers.StackHistorySerializer

class HostListAPIView(generics.ListAPIView):
    model = models.Host
    serializer_class = serializers.HostSerializer

    def get_queryset(self):
        return models.Host.objects.filter(stack__owner=self.request.user)

class StackHostsAPIView(HostListAPIView):
    parser_classes = (parsers.JSONParser,)

    def get_queryset(self):
        return models.Host.objects.filter(stack__pk=self.kwargs.get('pk'),
                                   stack__owner=self.request.user)

    def put(self, request, *args, **kwargs):
        '''
        Overriding PUT for a stack to be able to add additional
        hosts after a stack has already been created.
        '''

        stack = self.get_object()
        new_hosts = stack.add_hosts(request.DATA['hosts'])
        host_ids = [h.id for h in new_hosts]

        # Queue up stack creation and provisioning using Celery
        task_chain = (
            tasks.launch_hosts.si(stack.id, host_ids=host_ids) |
            tasks.update_metadata.si(stack.id, host_ids=host_ids) |
            tasks.tag_infrastructure.si(stack.id, host_ids=host_ids) |
            tasks.register_dns.si(stack.id, host_ids=host_ids) |
            tasks.ping.si(stack.id) |
            tasks.sync_all.si(stack.id) |
            tasks.highstate.si(stack.id, host_ids=host_ids) |
            tasks.orchestrate.si(stack.id, host_ids=host_ids) |
            tasks.finish_stack.si(stack.id)
        )

        # execute the chain
        task_chain()

        serializer = self.get_serializer(stack)
        return Response(serializer.data)

    def get(self, request, *args, **kwargs):
        '''
        Override get method to add additional host-specific info
        to the result that is looked up via salt.
        '''
        result = super(StackHostsAPIView, self).get(request, *args, **kwargs)
        if not result.data['results']:
            return result

        stack = request.user.stacks.get(id=kwargs.get('pk'))
        query_results = stack.query_hosts()

        # TODO: query_results are highly dependent on the underlying
        # salt-cloud driver and there's no guarantee that the result
        # format for AWS will be the same for Rackspace. In the future,
        # we should probably pass the results off to the cloud provider
        # implementation to format into a generic result for the user
        for host in result.data['results']:
            hostname = host['hostname']
            host['ec2_metadata'] = query_results[hostname]

        return result

class StackVolumesAPIView(VolumeListAPIView):

    def get_queryset(self):
        return Volume.objects.filter(stack__pk=self.kwargs.get('pk'))

class HostDetailAPIView(generics.RetrieveDestroyAPIView):
    model = models.Host
    serializer_class = serializers.HostSerializer

    def destroy(self, request, *args, **kwargs):
        '''
        Override the delete method to first terminate the host
        before destroying the object.
        '''
        # get the stack id for the host
        host = self.get_object()
        volume_ids = [v.id for v in host.volumes.all()]

        host.set_status(models.Host.DELETING, 'Deleting host.')

        # unregister DNS and destroy the host
        task_chain = (
            tasks.register_volume_delete.si(host.stack.id, host_ids=[host.id]) |
            tasks.unregister_dns.si(host.stack.id, host_ids=[host.id]) |
            tasks.destroy_hosts.si(host.stack.id, host_ids=[host.id])
        )

        # execute the chain
        task_chain()

        # Return the host while its deleting
        serializer = self.get_serializer(host)
        return Response(serializer.data)

<commit_msg>PI-183: removed some debug logic<commit_after>import logging
from collections import defaultdict
from operator import or_

import celery
from django.shortcuts import get_object_or_404

from rest_framework import (
    generics,
    parsers,
)
from rest_framework.response import Response

from core.exceptions import (
    ResourceConflict,
    BadRequest,
)

from volumes.api import VolumeListAPIView
from volumes.models import Volume
from blueprints.models import Blueprint

from . import tasks, models, serializers

logger = logging.getLogger(__name__)

class StackListAPIView(generics.ListCreateAPIView):
    '''
    TODO: Add docstring
    '''
    model = models.Stack
    serializer_class = serializers.StackSerializer
    parser_classes = (parsers.JSONParser,)

    def get_queryset(self):
        return self.request.user.stacks.all()

    def create(self, request, *args, **kwargs):
        '''
        Overriding create method to build roles and metadata objects for this Stack
        as well as generating the salt-cloud map that will be used to launch
        machines
        '''
        # make sure the user has a public key or they won't be able to SSH
        # later
        if not request.user.settings.public_key:
            raise BadRequest('You have not added a public key to your user '
                             'profile and will not be able to SSH in to any '
                             'machines. Please update your user profile '
                             'before continuing.')

        # Validate data
        errors = {}
        blueprint_id = request.DATA.pop('blueprint', '')
        title = request.DATA.get('title', '')
        description = request.DATA.get('description', '')
        properties = request.DATA.get('properties', {})

        # check for required blueprint
        if not blueprint_id:
            errors.setdefault('blueprint', []).append('This field is required.')
        else:
            try:
                blueprint = Blueprint.objects.get(pk=blueprint_id,
                                                  owner=request.user)
            except Blueprint.DoesNotExist:
                errors.setdefault('blueprint', []).append(
                    'Blueprint with id {0} does not exist.'.format(blueprint_id)
                )
            except ValueError:
                errors.setdefault('blueprint', []).append(
                    'This field must be an integer ID of an existing blueprint.'
                )

        if not title:
            errors.setdefault('title', []).append('This field is required.')

        if not description:
            errors.setdefault('description', []).append('This field is required.')

        # check for duplicates
        if models.Stack.objects.filter(owner=self.request.user, title=title).count():
            errors.setdefault('title', []).append(
                'A Stack with this title already exists in your account.'
            )

        if not isinstance(properties, dict):
            errors.setdefault('properties', []).append(
                'This field must be a JSON object.'
            )
        else:
            # user properties are not allowed to provide a __stackdio__ key
            if '__stackdio__' in properties:
                errors.setdefault('properties', []).append(
                    'The __stackdio__ key is reserved for system use.'
                )

        if errors:
            raise BadRequest(errors)

        # create the stack and related objects
        try:
            logger.debug(request.DATA)
            stack = models.Stack.objects.create_stack(request.user, blueprint, **request.DATA)
        except Exception, e:
            logger.exception(e)
            raise BadRequest(str(e))

        # set some defaults
        launch_stack = request.DATA.get('auto_launch', True)
        provision_stack = request.DATA.get('auto_provision', True)

        if launch_stack:
            # Queue up stack creation and provisioning using Celery
            task_list = [
                tasks.launch_hosts.si(stack.id),
                tasks.update_metadata.si(stack.id),
                tasks.tag_infrastructure.si(stack.id),
                tasks.register_dns.si(stack.id),
                tasks.ping.si(stack.id),
                tasks.sync_all.si(stack.id)
            ]

            # provisioning is optional (mainly useful for getting machines
            # up so you can play with salt states)
            if provision_stack:
                task_list.append(tasks.highstate.si(stack.id))
                task_list.append(tasks.orchestrate.si(stack.id))

            # always finish
            task_list.append(tasks.finish_stack.si(stack.id))

            logger.debug(task_list)
            task_chain = reduce(or_, task_list)

            # execute the chain
            task_chain(link_error=tasks.handle_error.s(stack.id))

        # return serialized stack object
        serializer = serializers.StackSerializer(stack, context={
            'request': request,
        })
        return Response(serializer.data)

class StackDetailAPIView(generics.RetrieveUpdateDestroyAPIView):
    model = models.Stack
    serializer_class = serializers.StackSerializer
    parser_classes = (parsers.JSONParser,)

    def get_object(self):
        return get_object_or_404(models.Stack, id=self.kwargs.get('pk'),
                                 owner=self.request.user)

    def destroy(self, request, *args, **kwargs):
        '''
        Overriding the delete method to make sure the stack
        is taken offline before being deleted
        '''
        # Update the status
        stack = self.get_object()
        msg = 'Stack will be removed upon successful termination ' \
              'of all machines'
        stack.set_status(models.Stack.DESTROYING, msg)

        # Queue up stack destroy tasks
        task_chain = (
            tasks.register_volume_delete.si(stack.id) |
            tasks.unregister_dns.si(stack.id) |
            tasks.destroy_hosts.si(stack.id)
        )

        # execute the chain
        task_chain()

        # Return the stack while its deleting
        serializer = self.get_serializer(stack)
        return Response(serializer.data)

    def put(self, request, *args, **kwargs):
        '''
        PUT request on a stack allows RPC-like actions to be called to
        interact with the stack. Request data is JSON, and must provide
        an `action` parameter. Actions may require additional arguments
        which may be provided via the `args` parameter.

        Valid actions: stop, start, restart, terminate
        '''

        stack = self.get_object()
        hosts = stack.get_hosts()
        host_count = hosts.count()
        action = request.DATA.get('action', None)
        args = request.DATA.get('args', [])

        if not action:
            raise BadRequest('action is a required parameter.')

        driver = stack.get_driver()
        available_actions = driver.get_available_actions()

        if action not in available_actions:
            raise BadRequest('action is not alowed. Only the following '
                             'actions are allowed: {0}'.format(', '.join(available_actions)))

        # In case of a launch action, the stack must not have any available
        # hosts (ie, the stack must have already been terminated.)
        if action == driver.ACTION_LAUNCH and host_count > 0:
            raise BadRequest('Launching a stack is only available when '
                             'the stack is in a terminated state. This '
                             'stack has %d hosts available.' % host_count)

        # All other actions require hosts to be available
        elif action != driver.ACTION_LAUNCH and host_count == 0:
            raise BadRequest('The submitted action requires the stack to have '
                             'available hosts. Perhaps you meant to run the '
                             'launch action instead.')

        # check the action against current states (e.g., starting can't happen
        # unless the hosts are in the stopped state.)
        # XXX: Assuming that host metadata is accurate here
        for host in hosts:
            if action == driver.ACTION_START and \
               host.state != driver.STATE_STOPPED:
                raise BadRequest('Start action requires all hosts to be in '
                                 'the stopped state first. At least one host '
                                 'is reporting an invalid '
                                 'state: %s' % host.state)
            if action == driver.ACTION_STOP and \
               host.state != driver.STATE_RUNNING:
                raise BadRequest('Stop action requires all hosts to be in '
                                 'the running state first. At least one host '
                                 'is reporting an invalid '
                                 'state: %s' % host.state)
            if action == driver.ACTION_TERMINATE and \
               host.state not in (driver.STATE_RUNNING, driver.STATE_STOPPED):
                raise BadRequest('Terminate action requires all hosts to be '
                                 'in the either the running or stopped state '
                                 'first. At least one host is reporting an '
                                 'invalid state: %s' % host.state)
            if action == driver.ACTION_PROVISION and \
               host.state not in (driver.STATE_RUNNING,):
                raise BadRequest('Provision action requires all hosts to be '
                                 'in the running state first. At least one '
                                 'host is reporting an '
                                 'invalid state: %s' % host.state)

        # Kick off the celery task for the given action
        stack.set_status(models.Stack.EXECUTING_ACTION,
                         'Stack is executing action \'{0}\''.format(action))

        # Keep track of the tasks we need to run for this execution
        task_list = []

        # FIXME: not generic
        if action in (driver.ACTION_STOP, driver.ACTION_TERMINATE):
            # Unregister DNS when executing the above actions
            task_list.append(tasks.unregister_dns.si(stack.id))

        # Launch is slightly different than other actions
        if action == driver.ACTION_LAUNCH:
            task_list.append(tasks.launch_hosts.si(stack.id))

        # Terminate should leverage salt-cloud or salt gets confused about
        # the state of things
        elif action == driver.ACTION_TERMINATE:
            task_list.append(tasks.destroy_hosts.si(stack.id, delete_stack=False))

        # Execute other actions
        else:
            task_list.append(tasks.execute_action.si(stack.id, action, *args))

        # Update the metadata after the action has been executed
        task_list.append(tasks.update_metadata.si(stack.id))

        # Launching requires us to tag the newly available infrastructure
        if action in (driver.ACTION_LAUNCH,):
            tasks.tag_infrastructure.si(stack.id)

        # These actions require new DNS and provisioning
        if action in (driver.ACTION_START, driver.ACTION_LAUNCH):
            # Register DNS when executing the above actions
            task_list.append(tasks.register_dns.si(stack.id))
            # Also reprovision just in case?
            # TODO: do we need this and are there cases where special things
            # need to happen after a host is restarted?
            task_list.append(tasks.ping.si(stack.id))
            task_list.append(tasks.sync_all.si(stack.id))
            task_list.append(tasks.highstate.si(stack.id))
            task_list.append(tasks.orchestrate.si(stack.id))

        task_list.append(tasks.finish_stack.si(stack.id))

        # chain together our tasks using the bitwise or operator
        task_chain = reduce(or_, task_list)

        # Update all host states
        hosts.update(state='actioning')

        # execute the chain
        task_chain()

        serializer = self.get_serializer(stack)
        return Response(serializer.data)

class StackHistoryList(generics.ListAPIView):
    model = models.StackHistory
    serializer_class = serializers.StackHistorySerializer

class HostListAPIView(generics.ListAPIView):
    model = models.Host
    serializer_class = serializers.HostSerializer

    def get_queryset(self):
        return models.Host.objects.filter(stack__owner=self.request.user)

class StackHostsAPIView(HostListAPIView):
    parser_classes = (parsers.JSONParser,)

    def get_queryset(self):
        return models.Host.objects.filter(stack__pk=self.kwargs.get('pk'),
                                   stack__owner=self.request.user)

    def put(self, request, *args, **kwargs):
        '''
        Overriding PUT for a stack to be able to add additional
        hosts after a stack has already been created.
        '''

        stack = self.get_object()
        new_hosts = stack.add_hosts(request.DATA['hosts'])
        host_ids = [h.id for h in new_hosts]

        # Queue up stack creation and provisioning using Celery
        task_chain = (
            tasks.launch_hosts.si(stack.id, host_ids=host_ids) |
            tasks.update_metadata.si(stack.id, host_ids=host_ids) |
            tasks.tag_infrastructure.si(stack.id, host_ids=host_ids) |
            tasks.register_dns.si(stack.id, host_ids=host_ids) |
            tasks.ping.si(stack.id) |
            tasks.sync_all.si(stack.id) |
            tasks.highstate.si(stack.id, host_ids=host_ids) |
            tasks.orchestrate.si(stack.id, host_ids=host_ids) |
            tasks.finish_stack.si(stack.id)
        )

        # execute the chain
        task_chain()

        serializer = self.get_serializer(stack)
        return Response(serializer.data)

    def get(self, request, *args, **kwargs):
        '''
        Override get method to add additional host-specific info
        to the result that is looked up via salt.
        '''
        result = super(StackHostsAPIView, self).get(request, *args, **kwargs)
        if not result.data['results']:
            return result

        stack = request.user.stacks.get(id=kwargs.get('pk'))
        query_results = stack.query_hosts()

        # TODO: query_results are highly dependent on the underlying
        # salt-cloud driver and there's no guarantee that the result
        # format for AWS will be the same for Rackspace. In the future,
        # we should probably pass the results off to the cloud provider
        # implementation to format into a generic result for the user
        for host in result.data['results']:
            hostname = host['hostname']
            host['ec2_metadata'] = query_results[hostname]

        return result

class StackVolumesAPIView(VolumeListAPIView):

    def get_queryset(self):
        return Volume.objects.filter(stack__pk=self.kwargs.get('pk'))

class HostDetailAPIView(generics.RetrieveDestroyAPIView):
    model = models.Host
    serializer_class = serializers.HostSerializer

    def destroy(self, request, *args, **kwargs):
        '''
        Override the delete method to first terminate the host
        before destroying the object.
        '''
        # get the stack id for the host
        host = self.get_object()
        volume_ids = [v.id for v in host.volumes.all()]

        host.set_status(models.Host.DELETING, 'Deleting host.')

        # unregister DNS and destroy the host
        task_chain = (
            tasks.register_volume_delete.si(host.stack.id, host_ids=[host.id]) |
            tasks.unregister_dns.si(host.stack.id, host_ids=[host.id]) |
            tasks.destroy_hosts.si(host.stack.id, host_ids=[host.id])
        )

        # execute the chain
        task_chain()

        # Return the host while its deleting
        serializer = self.get_serializer(host)
        return Response(serializer.data)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#                                                                             #
# Copyright (C) 2009  Renato Lima - Akretion                                  #
#                                                                             #
#This program is free software: you can redistribute it and/or modify         #
#it under the terms of the GNU Affero General Public License as published by  #
#the Free Software Foundation, either version 3 of the License, or            #
#(at your option) any later version.                                          #
#                                                                             #
#This program is distributed in the hope that it will be useful,              #
#but WITHOUT ANY WARRANTY; without even the implied warranty of               #
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
#GNU Affero General Public License for more details.                          #
#                                                                             #
#You should have received a copy of the GNU Affero General Public License     #
#along with this program.  If not, see <http://www.gnu.org/licenses/>.        #
###############################################################################

import time

from openerp import models, _
from openerp.exceptions import except_orm

class SaleOrder(models.Model):
    _inherit = 'sale.order'

    # TODO migrate to new API
    def _prepare_invoice(self, cr, uid, order, lines, context=None):
        """Prepare the dict of values to create the new invoice for a
           sale order. This method may be overridden to implement custom
           invoice generation (making sure to call super() to establish
           a clean extension chain).

           :param browse_record order: sale.order record to invoice
           :param list(int) line: list of invoice line IDs that must be
                                  attached to the invoice
           :return: dict of value to create() the invoice
        """
        result = super(SaleOrder, self)._prepare_invoice(
            cr, uid, order, lines, context)

        if order.carrier_id:
            result['carrier_id'] = order.carrier_id.id

        return result

    #TODO Inplement this in object stock.move method _picking_assign
    #def _prepare_order_picking(self, cr, uid, order, context=None):
    #    result = super(SaleOrder, self)._prepare_order_picking(
    #        cr, uid, order, context)
    #
    #    # FIXME - Confirmado bug do OpenERP
    #    # https://bugs.launchpad.net/bugs/1161138
    #    # Esse campo já deveria ser copiado pelo módulo nativo delivery
    #    result['incoterm'] = order.incoterm and order.incoterm.id or False
    #    return result

    # TODO migrate to new API
    def delivery_set(self, cr, uid, ids, context=None):
        #Copia do modulo delivery
        #Exceto pelo final que adiciona ao campo total do frete.
        grid_obj = self.pool.get('delivery.grid')
        carrier_obj = self.pool.get('delivery.carrier')

        for order in self.browse(cr, uid, ids, context=context):
            grid_id = carrier_obj.grid_get(cr, uid, [order.carrier_id.id],
            order.partner_shipping_id.id)

            if not grid_id:
                raise except_orm(_('No Grid Available!'),
                     _('No grid matching for this carrier!'))

            if not order.state in ('draft'):
                raise except_orm(_('Order not in Draft State!'),
                    _('The order state have to be draft to add delivery lines.'))

            grid = grid_obj.browse(cr, uid, grid_id, context=context)

            amount_freight = grid_obj.get_price(cr, uid, grid.id, order,
            time.strftime('%Y-%m-%d'), context)
            self.onchange_amount_freight(cr, uid, ids, amount_freight)
        return self.write(cr, uid, ids, {'amount_freight': amount_freight})
<commit_msg>Added api<commit_after># -*- coding: utf-8 -*-
###############################################################################
#                                                                             #
# Copyright (C) 2009  Renato Lima - Akretion                                  #
#                                                                             #
#This program is free software: you can redistribute it and/or modify         #
#it under the terms of the GNU Affero General Public License as published by  #
#the Free Software Foundation, either version 3 of the License, or            #
#(at your option) any later version.                                          #
#                                                                             #
#This program is distributed in the hope that it will be useful,              #
#but WITHOUT ANY WARRANTY; without even the implied warranty of               #
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
#GNU Affero General Public License for more details.                          #
#                                                                             #
#You should have received a copy of the GNU Affero General Public License     #
#along with this program.  If not, see <http://www.gnu.org/licenses/>.        #
###############################################################################

import time

from openerp import models, api, _
from openerp.exceptions import except_orm

class SaleOrder(models.Model):
    _inherit = 'sale.order'

    # TODO migrate to new API
    def _prepare_invoice(self, cr, uid, order, lines, context=None):
        """Prepare the dict of values to create the new invoice for a
           sale order. This method may be overridden to implement custom
           invoice generation (making sure to call super() to establish
           a clean extension chain).

           :param browse_record order: sale.order record to invoice
           :param list(int) line: list of invoice line IDs that must be
                                  attached to the invoice
           :return: dict of value to create() the invoice
        """
        result = super(SaleOrder, self)._prepare_invoice(
            cr, uid, order, lines, context)

        if order.carrier_id:
            result['carrier_id'] = order.carrier_id.id

        return result

    #TODO Inplement this in object stock.move method _picking_assign
    #def _prepare_order_picking(self, cr, uid, order, context=None):
    #    result = super(SaleOrder, self)._prepare_order_picking(
    #        cr, uid, order, context)
    #
    #    # FIXME - Confirmado bug do OpenERP
    #    # https://bugs.launchpad.net/bugs/1161138
    #    # Esse campo já deveria ser copiado pelo módulo nativo delivery
    #    result['incoterm'] = order.incoterm and order.incoterm.id or False
    #    return result

    # TODO migrate to new API
    def delivery_set(self, cr, uid, ids, context=None):
        #Copia do modulo delivery
        #Exceto pelo final que adiciona ao campo total do frete.
        grid_obj = self.pool.get('delivery.grid')
        carrier_obj = self.pool.get('delivery.carrier')

        for order in self.browse(cr, uid, ids, context=context):
            grid_id = carrier_obj.grid_get(cr, uid, [order.carrier_id.id],
            order.partner_shipping_id.id)

            if not grid_id:
                raise except_orm(_('No Grid Available!'),
                     _('No grid matching for this carrier!'))

            if not order.state in ('draft'):
                raise except_orm(_('Order not in Draft State!'),
                    _('The order state have to be draft to add delivery lines.'))

            grid = grid_obj.browse(cr, uid, grid_id, context=context)

            amount_freight = grid_obj.get_price(cr, uid, grid.id, order,
            time.strftime('%Y-%m-%d'), context)
            self.onchange_amount_freight(cr, uid, ids, amount_freight)
        return self.write(cr, uid, ids, {'amount_freight': amount_freight})
<|endoftext|>


<commit_before>'''
Common Themes
=============

Tenable.sc CRUD within pyTenable
--------------------------------

pyTenable allows for the ability to leverage both the naturalized inputs as well
as passing the raw parameters within the same structure.  In some cases this
doesn't seem immediately obvious, however allows for the ability to pass
parameters that either haven't yet been, or in some cases, may never be
interpreted by the library.  For example, in the alerts API, you could pass
the snake_cased ``always_exec_on_trigger`` or you could pass what the API
endpoint itself expects, which is ``executeOnEveryTrigger``.  The snake-cased
version expects a boolean value, which will be converted into the string value
that camelCased variant expects.  You'll see this behavior a lot throughout the
library, and is intended to allow you to sidestep most things should you need
to.  For example, in the alerts API again, you may not want to pass a trigger
as ``trigger=('sumip', '>=', '100')`` and instead pass as the parameters that
are to be written into the JSON request:
``triggerName='sumip', triggerOperator='>=', triggerValue='100'``.  Both of
these methods will produce the same JSON request, and the the option is yours
to use the right way for the job.  Along these same lines, its possible to see
how the JSON documents are being constructed by simply looking at the
``_constructor`` methods for each APIEndpoint class.  If pyTenable is getting in
your way with the, you can almost always sidestep it and pass the exact
dictionary you wish to pass on to the API.

Schedule Dictionaries
---------------------

A dictionary detailing the repeating schedule within Tenable.sc.  This
dictionary consists of 1 or 3 parameters, depending on the type of schedule.
In all of the definitions except ``ical``, a  single parameter of ``type`` is
passed with lone of the following values: ``ical``, ``never``, ``rollover``, and
``template``.  If no document is specified, then the default of ``never`` is
assumed.  For repeating scans, you'll have to use the type of ``ical`` and also
specify the ``start`` and ``rrule`` parameters as well.  The ``start`` parameter
is an `iCal DateTime Form #3 <https://tools.ietf.org/html/rfc5545#section-3.3.5>`_
formatted string specifying the date and time in which to start the repeating
event.  The ``rrule`` parameter is an
`iCal Recurrance Rule <https://tools.ietf.org/html/rfc5545#section-3.3.10>`_
formatted string.

* Example Never Declaration:

.. code-block:: python

    {'type': 'never'}

* Example daily event starting at 9am Eastern

.. code-block:: python

    {
        'type': 'ical',
        'start': 'TZID=America/New York:20190214T090000',
        'rrule': 'FREQ=DAILY;INTERVAL=1'
    }

* Example weekly event every Saturday at 8:30pm Eastern

.. code-block:: python

    {
        'type': 'ical',
        'start': 'TZID=America/New York:20190214T203000',
        'rrule': 'FREQ=WEEKLY;BYDAY=SA;INTERVAL=1'
    }

There are detailed instructions in the RFC documentation on how to construct
these recurrence rules.  Further there are some packages out there to aid in
converting more human-readable text into recurrence rules, such as the
`recurrent package <https://pypi.org/project/recurrent/>`_ for example.
'''
from tenable.base import APIEndpoint, APIResultsIterator

class SCEndpoint(APIEndpoint):
    def _combo_expansion(self, item):
        '''
        Expands the asset combination expressions from nested tuples to the
        nested dictionary structure that's expected.

        Args:
            item

        Returns:
            dict: The dictionary structure of the expanded asset list combinations.
        '''

        # the operator conversion dictionary.  The UI uses "and", "or", and
        # "not" whereas the API uses "intersection", "union", and "compliment".
        # if the user is passing us the tuples, lets assume that they are using
        # the UI definitions and not the API ones.
        oper = {
            'and': 'intersection',
            'or': 'union',
            'not': 'complement'
        }

        # some simple checking to ensure that we are being passed good data
        # before we expand the tuple.
        if len(item) < 2 or len(item) > 3:
            raise TypeError('{} must be exactly 1 operator and 1-2 items'.format(item))
        self._check('operator', item[0], str, choices=oper.keys())
        self._check('operand1', item[1], [int, tuple])
        if len(item) == 3:
            self._check('operand2', item[2], [int, tuple])

        resp = {'operator': oper[item[0].lower()]}

        # we need to expand the operand.  If the item is a nested tuple, then
        # we will call ourselves and pass the tuple.  If not, then we will
        # simply return a dictionary with the id value set to the integer that
        # was passed.
        if isinstance(item[1], tuple):
            resp['operand1'] = self._combo_expansion(item[1])
        else:
            resp['operand1'] = {'id': str(item[1])}

        # if there are 2 operators in the tuple, then we will want to expand the
        # second one as well. If the item is a nested tuple, then we will call
        # ourselves and pass the tuple.  If not, then we will simply return a
        # dictionary with the id value set to the integer that was passed.
        if len(item) == 3:
            if isinstance(item[2], tuple):
                resp['operand2'] = self._combo_expansion(item[2])
            else:
                resp['operand2'] = {'id': str(item[2])}

        # return the response to the caller.
        return resp

    def _schedule_constructor(self, item):
        '''
        Handles creation of the schedule sub-document.
        '''
        self._check('schedule:item', item, dict)
        resp = {'type': self._check('schedule:type', item['type'], str,
            choices=['ical', 'dependent', 'never', 'rollover', 'template', 'now'],
            default='never')}

        if resp['type'] == 'ical' and 'start' in item and 'repeatRule' in item:
            resp['start'] = self._check('schedule:start', item['start'], str)
            resp['repeatRule'] = self._check(
                'schedule:repeatRule', item['repeatRule'], str)
        return resp

class SCResultsIterator(APIResultsIterator):
    pass
<commit_msg>Added a generic page getter to the base iterator<commit_after>'''
Common Themes
=============

Tenable.sc CRUD within pyTenable
--------------------------------

pyTenable allows for the ability to leverage both the naturalized inputs as well
as passing the raw parameters within the same structure.  In some cases this
doesn't seem immediately obvious, however allows for the ability to pass
parameters that either haven't yet been, or in some cases, may never be
interpreted by the library.  For example, in the alerts API, you could pass
the snake_cased ``always_exec_on_trigger`` or you could pass what the API
endpoint itself expects, which is ``executeOnEveryTrigger``.  The snake-cased
version expects a boolean value, which will be converted into the string value
that camelCased variant expects.  You'll see this behavior a lot throughout the
library, and is intended to allow you to sidestep most things should you need
to.  For example, in the alerts API again, you may not want to pass a trigger
as ``trigger=('sumip', '>=', '100')`` and instead pass as the parameters that
are to be written into the JSON request:
``triggerName='sumip', triggerOperator='>=', triggerValue='100'``.  Both of
these methods will produce the same JSON request, and the the option is yours
to use the right way for the job.  Along these same lines, its possible to see
how the JSON documents are being constructed by simply looking at the
``_constructor`` methods for each APIEndpoint class.  If pyTenable is getting in
your way with the, you can almost always sidestep it and pass the exact
dictionary you wish to pass on to the API.

Schedule Dictionaries
---------------------

A dictionary detailing the repeating schedule within Tenable.sc.  This
dictionary consists of 1 or 3 parameters, depending on the type of schedule.
In all of the definitions except ``ical``, a  single parameter of ``type`` is
passed with lone of the following values: ``ical``, ``never``, ``rollover``, and
``template``.  If no document is specified, then the default of ``never`` is
assumed.  For repeating scans, you'll have to use the type of ``ical`` and also
specify the ``start`` and ``rrule`` parameters as well.  The ``start`` parameter
is an `iCal DateTime Form #3 <https://tools.ietf.org/html/rfc5545#section-3.3.5>`_
formatted string specifying the date and time in which to start the repeating
event.  The ``rrule`` parameter is an
`iCal Recurrance Rule <https://tools.ietf.org/html/rfc5545#section-3.3.10>`_
formatted string.

* Example Never Declaration:

.. code-block:: python

    {'type': 'never'}

* Example daily event starting at 9am Eastern

.. code-block:: python

    {
        'type': 'ical',
        'start': 'TZID=America/New York:20190214T090000',
        'rrule': 'FREQ=DAILY;INTERVAL=1'
    }

* Example weekly event every Saturday at 8:30pm Eastern

.. code-block:: python

    {
        'type': 'ical',
        'start': 'TZID=America/New York:20190214T203000',
        'rrule': 'FREQ=WEEKLY;BYDAY=SA;INTERVAL=1'
    }

There are detailed instructions in the RFC documentation on how to construct
these recurrence rules.  Further there are some packages out there to aid in
converting more human-readable text into recurrence rules, such as the
`recurrent package <https://pypi.org/project/recurrent/>`_ for example.
'''
from tenable.base import APIEndpoint, APIResultsIterator

class SCEndpoint(APIEndpoint):
    def _combo_expansion(self, item):
        '''
        Expands the asset combination expressions from nested tuples to the
        nested dictionary structure that's expected.

        Args:
            item

        Returns:
            dict: The dictionary structure of the expanded asset list combinations.
        '''

        # the operator conversion dictionary.  The UI uses "and", "or", and
        # "not" whereas the API uses "intersection", "union", and "compliment".
        # if the user is passing us the tuples, lets assume that they are using
        # the UI definitions and not the API ones.
        oper = {
            'and': 'intersection',
            'or': 'union',
            'not': 'complement'
        }

        # some simple checking to ensure that we are being passed good data
        # before we expand the tuple.
        if len(item) < 2 or len(item) > 3:
            raise TypeError('{} must be exactly 1 operator and 1-2 items'.format(item))
        self._check('operator', item[0], str, choices=oper.keys())
        self._check('operand1', item[1], [int, tuple])
        if len(item) == 3:
            self._check('operand2', item[2], [int, tuple])

        resp = {'operator': oper[item[0].lower()]}

        # we need to expand the operand.  If the item is a nested tuple, then
        # we will call ourselves and pass the tuple.  If not, then we will
        # simply return a dictionary with the id value set to the integer that
        # was passed.
        if isinstance(item[1], tuple):
            resp['operand1'] = self._combo_expansion(item[1])
        else:
            resp['operand1'] = {'id': str(item[1])}

        # if there are 2 operators in the tuple, then we will want to expand the
        # second one as well. If the item is a nested tuple, then we will call
        # ourselves and pass the tuple.  If not, then we will simply return a
        # dictionary with the id value set to the integer that was passed.
        if len(item) == 3:
            if isinstance(item[2], tuple):
                resp['operand2'] = self._combo_expansion(item[2])
            else:
                resp['operand2'] = {'id': str(item[2])}

        # return the response to the caller.
        return resp

    def _schedule_constructor(self, item):
        '''
        Handles creation of the schedule sub-document.
        '''
        self._check('schedule:item', item, dict)
        resp = {'type': self._check('schedule:type', item['type'], str,
            choices=['ical', 'dependent', 'never', 'rollover', 'template', 'now'],
            default='never')}

        if resp['type'] == 'ical' and 'start' in item and 'repeatRule' in item:
            resp['start'] = self._check('schedule:start', item['start'], str)
            resp['repeatRule'] = self._check(
                'schedule:repeatRule', item['repeatRule'], str)
        return resp

class SCResultsIterator(APIResultsIterator):
    def _get_page(self):
        '''
        Retreives the next page of results when the current page has been
        exhausted.
        '''
        # First we need to see if there is a page limit and if there is, have
        # we run into that limit.  If we have, then return a StopIteration
        # exception.
        if self._pages_total and self._pages_requested >= self._pages_total:
            raise StopIteration()

        # Now we need to do is construct the query with the current offset
        # and limits
        query = self._query
        query['startOffset'] = self._offset
        query['endOffset'] = self._limit + self._offset

        # Lets actually call the API for the data at this point.
        resp = self._api.get(self._resource, params=query).json()

        # Now that we have the response, lets reset any counters we need to,
        # and increment things like the page counter, offset, etc.
        self.page_count = 0
        self._pages_requested += 1
        self._offset += self._limit
        self._raw = resp
        self.page = resp['response']

        # As no total is returned via the API, we will simply need to re-compute
        # the total to be N+1 every page as long as the page of data is equal to
        # the limit.  If we ever get a page of data that is less than the limit,
        # then we will set the total to be the count + page length.
        if len(resp['response']) < self._limit:
            self.total = self.count + len(resp['response'])
        else:
            self.total = self.count + self._limit + 1<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Packages
import codecs
from pyquery import PyQuery as pq
import re

def xmltoDatabase():
# Open xml list and future candidates database .csv
# This is the updated list (nov 24), replacing the candidates-list_old.xml (nov 18)
    candidates = codecs.open("candidates-database.csv", "w", "utf-8")
    candidates.write(('"%s","%s","%s","%s","%s","%s","%s"\n') % ("name", "sex", "election_type", "district", "party", "initials", "position"))
    page = pq(filename = "candidates-list.xml")
    district = ""
    election_type = ""
    party = ""
    initials = ""
    position = ""
    sex = ""
    substitute = False

# Extracting informations
    for line in page("texto p"):
        name = ""
        if not line.text:
            continue
# Differentiating section titles from candidates' name
        if line.text[:5] == "JUNTA":
            district = " ".join(line.text.strip().split(" ")[3:])
        if line.text[:8] == "CONGRESO":
            election_type = "congress"
        if line.text[:6] == "SENADO":
            election_type = "senate"
        if pq(line).attr("class") == "centro_redonda":
            party = line.text.split("(")[0]
            party = " ".join(party.strip().split(" ")[1:])
            substitute = False
            if "(" in line.text:
                initials = line.text.strip().split("(")[1][:-1]
            else:
                initials = ""
        # Extracting informations on candidates (name, sex, list position, party)
        if pq(line).attr("class")[:7] == "parrafo" and line.text[0] in "0123456789" and line.text != line.text.upper():
            position = line.text.strip().split(" ")[0][:-1]
            if line.text.split(" ")[1] == "Don":
                sex = 0
                name = " ".join(line.text.strip().split(" ")[2:])[:-1]
            elif line.text.split(" ")[1] == u"Doña":
                sex = 1
                name = " ".join(line.text.strip().split(" ")[2:])[:-1]
            else:
                sex = ""
                name = " ".join(line.text.strip().split(" ")[1:])[:-1]
            if "Independiente" in name:
                party = "Independiente"
        elif line.text[0] in "0123456789" and line.text == line.text.upper():
            party = line.text.split("(")[0]
            party = " ".join(party.strip().split(" ")[1:])
            if "(" in line.text:
                initials = line.text.strip().split("(")[1][:-1]
            else:
                initials = ""
        else:
            name = ""
        if line.text[:8] == "Suplente":
            if line.text[-1] == ":":
                substitute = True
            else:
                continue
        if "(U" in name:
            name = name.split("(")[0]
        if "(" in name.split("(")[:-1]:
            name = " ".join(name.split("(")[:-1])
        # Writing informations in database, putting aside substitutes candidates
        if substitute == True:
            name = ""
        if checkParties(initials) == True and name != "":
            if sex == "":
                sex = addInformations(name)
            candidates.write(('"%s","%s","%s","%s","%s","%s","%s"\n') % (cleanText(name), sex, election_type, district, party, initials, position))
    candidates.close()

# Keeping only candidates from selected parties
def checkParties(party):
    tab = [u"C's",u"C´s",u"EN COMÚ",u"ERC",u"EUPV",u"EH Bildu",u"IU",u"UNIO.CAT",u"PNV",u"PODEMOS",u"PP",u"PS",u"UPyD"]
    for line in tab:
        if line in party:
            return True
    return False

# Adding informations from old database (where sex was specified using Don/Doña)
def addInformations(name):
    data = codecs.open("candidates-database_old.csv","r","utf-8")
    sex = ""
    for line in data:
        infos = line.strip().split(',')
        if not name in infos[0]:
            continue
        else:
            sex = infos[1][1:-1]
            break
    data.close()
    return(sex)

# Removing " from text (to avoid confusion with .csv separators)
def cleanText(text):
    text = re.sub('["]', " ", text)
    return(text)

xmltoDatabase()
<commit_msg>Delete extract-candidates.py<commit_after><|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""TFRecord sources and sinks."""

from __future__ import absolute_import

import codecs
import logging
import struct
from builtins import object
from functools import partial

import crcmod

from apache_beam import coders
from apache_beam.io import filebasedsink
from apache_beam.io.filebasedsource import FileBasedSource
from apache_beam.io.filebasedsource import ReadAllFiles
from apache_beam.io.filesystem import CompressionTypes
from apache_beam.io.iobase import Read
from apache_beam.io.iobase import Write
from apache_beam.transforms import PTransform

__all__ = ['ReadFromTFRecord', 'WriteToTFRecord']

def _default_crc32c_fn(value):
  """Calculates crc32c by either snappy or crcmod based on installation."""

  if not _default_crc32c_fn.fn:
    try:
      import snappy  # pylint: disable=import-error
      # Support multiple versions of python-snappy:
      # https://github.com/andrix/python-snappy/pull/53
      if getattr(snappy, '_crc32c', None):
        _default_crc32c_fn.fn = snappy._crc32c  # pylint: disable=protected-access
      else:
        _default_crc32c_fn.fn = snappy._snappy._crc32c  # pylint: disable=protected-access
    except ImportError:
      logging.warning('Couldn\'t find python-snappy so the implementation of '
                      '_TFRecordUtil._masked_crc32c is not as fast as it could '
                      'be.')
      _default_crc32c_fn.fn = crcmod.predefined.mkPredefinedCrcFun('crc-32c')
  return _default_crc32c_fn.fn(value)

_default_crc32c_fn.fn = None

class _TFRecordUtil(object):
  """Provides basic TFRecord encoding/decoding with consistency checks.

  For detailed TFRecord format description see:
    https://www.tensorflow.org/versions/r1.11/api_guides/python/python_io#TFRecords_Format_Details

  Note that masks and length are represented in LittleEndian order.
  """

  @classmethod
  def _masked_crc32c(cls, value, crc32c_fn=_default_crc32c_fn):
    """Compute a masked crc32c checksum for a value.

    Args:
      value: A string for which we compute the crc.
      crc32c_fn: A function that can compute a crc32c.
        This is a performance hook that also helps with testing. Callers are
        not expected to make use of it directly.
    Returns:
      Masked crc32c checksum.
    """

    crc = crc32c_fn(value)
    return (((crc >> 15) | (crc << 17)) + 0xa282ead8) & 0xffffffff

  @staticmethod
  def encoded_num_bytes(record):
    """Return the number of bytes consumed by a record in its encoded form."""
    # 16 = 8 (Length) + 4 (crc of length) + 4 (crc of data)
    return len(record) + 16

  @classmethod
  def write_record(cls, file_handle, value):
    """Encode a value as a TFRecord.

    Args:
      file_handle: The file to write to.
      value: A string content of the record.
    """
    encoded_length = struct.pack('<Q', len(value))
    file_handle.write('{}{}{}{}'.format(
        encoded_length,
        struct.pack('<I', cls._masked_crc32c(encoded_length)),  #
        value,
        struct.pack('<I', cls._masked_crc32c(value))))

  @classmethod
  def read_record(cls, file_handle):
    """Read a record from a TFRecords file.

    Args:
      file_handle: The file to read from.
    Returns:
      None if EOF is reached; the paylod of the record otherwise.
    Raises:
      ValueError: If file appears to not be a valid TFRecords file.
    """
    buf_length_expected = 12
    buf = file_handle.read(buf_length_expected)
    if not buf:
      return None  # EOF Reached.

    # Validate all length related payloads.
    if len(buf) != buf_length_expected:
      raise ValueError('Not a valid TFRecord. Fewer than %d bytes: %s' %
                       (buf_length_expected, codecs.encode(buf, 'hex')))
    length, length_mask_expected = struct.unpack('<QI', buf)
    length_mask_actual = cls._masked_crc32c(buf[:8])
    if length_mask_actual != length_mask_expected:
      raise ValueError('Not a valid TFRecord. Mismatch of length mask: %s' %
                       codecs.encode(buf, 'hex'))

    # Validate all data related payloads.
    buf_length_expected = length + 4
    buf = file_handle.read(buf_length_expected)
    if len(buf) != buf_length_expected:
      raise ValueError('Not a valid TFRecord. Fewer than %d bytes: %s' %
                       (buf_length_expected, codecs.encode(buf, 'hex')))
    data, data_mask_expected = struct.unpack('<%dsI' % length, buf)
    data_mask_actual = cls._masked_crc32c(data)
    if data_mask_actual != data_mask_expected:
      raise ValueError('Not a valid TFRecord. Mismatch of data mask: %s' %
                       codecs.encode(buf, 'hex'))

    # All validation checks passed.
    return data

class _TFRecordSource(FileBasedSource):
  """A File source for reading files of TFRecords.

  For detailed TFRecords format description see:
    https://www.tensorflow.org/versions/r1.11/api_guides/python/python_io#TFRecords_Format_Details
  """

  def __init__(self,
               file_pattern,
               coder,
               compression_type,
               validate):
    """Initialize a TFRecordSource.  See ReadFromTFRecord for details."""
    super(_TFRecordSource, self).__init__(
        file_pattern=file_pattern,
        compression_type=compression_type,
        splittable=False,
        validate=validate)
    self._coder = coder

  def read_records(self, file_name, offset_range_tracker):
    if offset_range_tracker.start_position():
      raise ValueError('Start position not 0:%s' %
                       offset_range_tracker.start_position())

    current_offset = offset_range_tracker.start_position()
    with self.open_file(file_name) as file_handle:
      while True:
        if not offset_range_tracker.try_claim(current_offset):
          raise RuntimeError('Unable to claim position: %s' % current_offset)
        record = _TFRecordUtil.read_record(file_handle)
        if record is None:
          return  # Reached EOF
        else:
          current_offset += _TFRecordUtil.encoded_num_bytes(record)
          yield self._coder.decode(record)

def _create_tfrecordio_source(
    file_pattern=None, coder=None, compression_type=None):
  # We intentionally disable validation for ReadAll pattern so that reading does
  # not fail for globs (elements) that are empty.
  return _TFRecordSource(file_pattern, coder, compression_type,
                         validate=False)

class ReadAllFromTFRecord(PTransform):
  """A ``PTransform`` for reading a ``PCollection`` of TFRecord files."""

  def __init__(
      self,
      coder=coders.BytesCoder(),
      compression_type=CompressionTypes.AUTO,
      **kwargs):
    """Initialize the ``ReadAllFromTFRecord`` transform.

    Args:
      coder: Coder used to decode each record.
      compression_type: Used to handle compressed input files. Default value
          is CompressionTypes.AUTO, in which case the file_path's extension will
          be used to detect the compression.
      **kwargs: optional args dictionary. These are passed through to parent
        constructor.
    """
    super(ReadAllFromTFRecord, self).__init__(**kwargs)
    source_from_file = partial(
        _create_tfrecordio_source, compression_type=compression_type,
        coder=coder)
    # Desired and min bundle sizes do not matter since TFRecord files are
    # unsplittable.
    self._read_all_files = ReadAllFiles(
        splittable=False, compression_type=compression_type,
        desired_bundle_size=0, min_bundle_size=0,
        source_from_file=source_from_file)

  def expand(self, pvalue):
    return pvalue | 'ReadAllFiles' >> self._read_all_files

class ReadFromTFRecord(PTransform):
  """Transform for reading TFRecord sources."""

  def __init__(self,
               file_pattern,
               coder=coders.BytesCoder(),
               compression_type=CompressionTypes.AUTO,
               validate=True,
               **kwargs):
    """Initialize a ReadFromTFRecord transform.

    Args:
      file_pattern: A file glob pattern to read TFRecords from.
      coder: Coder used to decode each record.
      compression_type: Used to handle compressed input files. Default value
          is CompressionTypes.AUTO, in which case the file_path's extension will
          be used to detect the compression.
      validate: Boolean flag to verify that the files exist during the pipeline
          creation time.
      **kwargs: optional args dictionary. These are passed through to parent
        constructor.

    Returns:
      A ReadFromTFRecord transform object.
    """
    super(ReadFromTFRecord, self).__init__(**kwargs)
    self._source = _TFRecordSource(file_pattern, coder, compression_type,
                                   validate)

  def expand(self, pvalue):
    return pvalue.pipeline | Read(self._source)

class _TFRecordSink(filebasedsink.FileBasedSink):
  """Sink for writing TFRecords files.

  For detailed TFRecord format description see:
    https://www.tensorflow.org/versions/r1.11/api_guides/python/python_io#TFRecords_Format_Details
  """

  def __init__(self, file_path_prefix, coder, file_name_suffix, num_shards,
               shard_name_template, compression_type):
    """Initialize a TFRecordSink. See WriteToTFRecord for details."""

    super(_TFRecordSink, self).__init__(
        file_path_prefix=file_path_prefix,
        coder=coder,
        file_name_suffix=file_name_suffix,
        num_shards=num_shards,
        shard_name_template=shard_name_template,
        mime_type='application/octet-stream',
        compression_type=compression_type)

  def write_encoded_record(self, file_handle, value):
    _TFRecordUtil.write_record(file_handle, value)

class WriteToTFRecord(PTransform):
  """Transform for writing to TFRecord sinks."""

  def __init__(self,
               file_path_prefix,
               coder=coders.BytesCoder(),
               file_name_suffix='',
               num_shards=0,
               shard_name_template=None,
               compression_type=CompressionTypes.AUTO,
               **kwargs):
    """Initialize WriteToTFRecord transform.

    Args:
      file_path_prefix: The file path to write to. The files written will begin
        with this prefix, followed by a shard identifier (see num_shards), and
        end in a common extension, if given by file_name_suffix.
      coder: Coder used to encode each record.
      file_name_suffix: Suffix for the files written.
      num_shards: The number of files (shards) used for output. If not set, the
        default value will be used.
      shard_name_template: A template string containing placeholders for
        the shard number and shard count. When constructing a filename for a
        particular shard number, the upper-case letters 'S' and 'N' are
        replaced with the 0-padded shard number and shard count respectively.
        This argument can be '' in which case it behaves as if num_shards was
        set to 1 and only one file will be generated. The default pattern used
        is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
      compression_type: Used to handle compressed output files. Typical value
          is CompressionTypes.AUTO, in which case the file_path's extension will
          be used to detect the compression.
      **kwargs: Optional args dictionary. These are passed through to parent
        constructor.

    Returns:
      A WriteToTFRecord transform object.
    """
    super(WriteToTFRecord, self).__init__(**kwargs)
    self._sink = _TFRecordSink(file_path_prefix, coder, file_name_suffix,
                               num_shards, shard_name_template,
                               compression_type)

  def expand(self, pcoll):
    return pcoll | Write(self._sink)
<commit_msg>[BEAM-5879 ] Make write_record() in tfrecordio.py py3 compatible (#6953)<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""TFRecord sources and sinks."""

from __future__ import absolute_import

import codecs
import logging
import struct
from builtins import object
from functools import partial

import crcmod

from apache_beam import coders
from apache_beam.io import filebasedsink
from apache_beam.io.filebasedsource import FileBasedSource
from apache_beam.io.filebasedsource import ReadAllFiles
from apache_beam.io.filesystem import CompressionTypes
from apache_beam.io.iobase import Read
from apache_beam.io.iobase import Write
from apache_beam.transforms import PTransform

__all__ = ['ReadFromTFRecord', 'WriteToTFRecord']

def _default_crc32c_fn(value):
  """Calculates crc32c by either snappy or crcmod based on installation."""

  if not _default_crc32c_fn.fn:
    try:
      import snappy  # pylint: disable=import-error
      # Support multiple versions of python-snappy:
      # https://github.com/andrix/python-snappy/pull/53
      if getattr(snappy, '_crc32c', None):
        _default_crc32c_fn.fn = snappy._crc32c  # pylint: disable=protected-access
      else:
        _default_crc32c_fn.fn = snappy._snappy._crc32c  # pylint: disable=protected-access
    except ImportError:
      logging.warning('Couldn\'t find python-snappy so the implementation of '
                      '_TFRecordUtil._masked_crc32c is not as fast as it could '
                      'be.')
      _default_crc32c_fn.fn = crcmod.predefined.mkPredefinedCrcFun('crc-32c')
  return _default_crc32c_fn.fn(value)

_default_crc32c_fn.fn = None

class _TFRecordUtil(object):
  """Provides basic TFRecord encoding/decoding with consistency checks.

  For detailed TFRecord format description see:
    https://www.tensorflow.org/versions/r1.11/api_guides/python/python_io#TFRecords_Format_Details

  Note that masks and length are represented in LittleEndian order.
  """

  @classmethod
  def _masked_crc32c(cls, value, crc32c_fn=_default_crc32c_fn):
    """Compute a masked crc32c checksum for a value.

    Args:
      value: A string for which we compute the crc.
      crc32c_fn: A function that can compute a crc32c.
        This is a performance hook that also helps with testing. Callers are
        not expected to make use of it directly.
    Returns:
      Masked crc32c checksum.
    """

    crc = crc32c_fn(value)
    return (((crc >> 15) | (crc << 17)) + 0xa282ead8) & 0xffffffff

  @staticmethod
  def encoded_num_bytes(record):
    """Return the number of bytes consumed by a record in its encoded form."""
    # 16 = 8 (Length) + 4 (crc of length) + 4 (crc of data)
    return len(record) + 16

  @classmethod
  def write_record(cls, file_handle, value):
    """Encode a value as a TFRecord.

    Args:
      file_handle: The file to write to.
      value: A string content of the record.
    """
    encoded_length = struct.pack(b'<Q', len(value))
    file_handle.write(b''.join([
        encoded_length,
        struct.pack(b'<I', cls._masked_crc32c(encoded_length)),
        value,
        struct.pack(b'<I', cls._masked_crc32c(value))
    ]))

  @classmethod
  def read_record(cls, file_handle):
    """Read a record from a TFRecords file.

    Args:
      file_handle: The file to read from.
    Returns:
      None if EOF is reached; the paylod of the record otherwise.
    Raises:
      ValueError: If file appears to not be a valid TFRecords file.
    """
    buf_length_expected = 12
    buf = file_handle.read(buf_length_expected)
    if not buf:
      return None  # EOF Reached.

    # Validate all length related payloads.
    if len(buf) != buf_length_expected:
      raise ValueError('Not a valid TFRecord. Fewer than %d bytes: %s' %
                       (buf_length_expected, codecs.encode(buf, 'hex')))
    length, length_mask_expected = struct.unpack('<QI', buf)
    length_mask_actual = cls._masked_crc32c(buf[:8])
    if length_mask_actual != length_mask_expected:
      raise ValueError('Not a valid TFRecord. Mismatch of length mask: %s' %
                       codecs.encode(buf, 'hex'))

    # Validate all data related payloads.
    buf_length_expected = length + 4
    buf = file_handle.read(buf_length_expected)
    if len(buf) != buf_length_expected:
      raise ValueError('Not a valid TFRecord. Fewer than %d bytes: %s' %
                       (buf_length_expected, codecs.encode(buf, 'hex')))
    data, data_mask_expected = struct.unpack('<%dsI' % length, buf)
    data_mask_actual = cls._masked_crc32c(data)
    if data_mask_actual != data_mask_expected:
      raise ValueError('Not a valid TFRecord. Mismatch of data mask: %s' %
                       codecs.encode(buf, 'hex'))

    # All validation checks passed.
    return data

class _TFRecordSource(FileBasedSource):
  """A File source for reading files of TFRecords.

  For detailed TFRecords format description see:
    https://www.tensorflow.org/versions/r1.11/api_guides/python/python_io#TFRecords_Format_Details
  """

  def __init__(self,
               file_pattern,
               coder,
               compression_type,
               validate):
    """Initialize a TFRecordSource.  See ReadFromTFRecord for details."""
    super(_TFRecordSource, self).__init__(
        file_pattern=file_pattern,
        compression_type=compression_type,
        splittable=False,
        validate=validate)
    self._coder = coder

  def read_records(self, file_name, offset_range_tracker):
    if offset_range_tracker.start_position():
      raise ValueError('Start position not 0:%s' %
                       offset_range_tracker.start_position())

    current_offset = offset_range_tracker.start_position()
    with self.open_file(file_name) as file_handle:
      while True:
        if not offset_range_tracker.try_claim(current_offset):
          raise RuntimeError('Unable to claim position: %s' % current_offset)
        record = _TFRecordUtil.read_record(file_handle)
        if record is None:
          return  # Reached EOF
        else:
          current_offset += _TFRecordUtil.encoded_num_bytes(record)
          yield self._coder.decode(record)

def _create_tfrecordio_source(
    file_pattern=None, coder=None, compression_type=None):
  # We intentionally disable validation for ReadAll pattern so that reading does
  # not fail for globs (elements) that are empty.
  return _TFRecordSource(file_pattern, coder, compression_type,
                         validate=False)

class ReadAllFromTFRecord(PTransform):
  """A ``PTransform`` for reading a ``PCollection`` of TFRecord files."""

  def __init__(
      self,
      coder=coders.BytesCoder(),
      compression_type=CompressionTypes.AUTO,
      **kwargs):
    """Initialize the ``ReadAllFromTFRecord`` transform.

    Args:
      coder: Coder used to decode each record.
      compression_type: Used to handle compressed input files. Default value
          is CompressionTypes.AUTO, in which case the file_path's extension will
          be used to detect the compression.
      **kwargs: optional args dictionary. These are passed through to parent
        constructor.
    """
    super(ReadAllFromTFRecord, self).__init__(**kwargs)
    source_from_file = partial(
        _create_tfrecordio_source, compression_type=compression_type,
        coder=coder)
    # Desired and min bundle sizes do not matter since TFRecord files are
    # unsplittable.
    self._read_all_files = ReadAllFiles(
        splittable=False, compression_type=compression_type,
        desired_bundle_size=0, min_bundle_size=0,
        source_from_file=source_from_file)

  def expand(self, pvalue):
    return pvalue | 'ReadAllFiles' >> self._read_all_files

class ReadFromTFRecord(PTransform):
  """Transform for reading TFRecord sources."""

  def __init__(self,
               file_pattern,
               coder=coders.BytesCoder(),
               compression_type=CompressionTypes.AUTO,
               validate=True,
               **kwargs):
    """Initialize a ReadFromTFRecord transform.

    Args:
      file_pattern: A file glob pattern to read TFRecords from.
      coder: Coder used to decode each record.
      compression_type: Used to handle compressed input files. Default value
          is CompressionTypes.AUTO, in which case the file_path's extension will
          be used to detect the compression.
      validate: Boolean flag to verify that the files exist during the pipeline
          creation time.
      **kwargs: optional args dictionary. These are passed through to parent
        constructor.

    Returns:
      A ReadFromTFRecord transform object.
    """
    super(ReadFromTFRecord, self).__init__(**kwargs)
    self._source = _TFRecordSource(file_pattern, coder, compression_type,
                                   validate)

  def expand(self, pvalue):
    return pvalue.pipeline | Read(self._source)

class _TFRecordSink(filebasedsink.FileBasedSink):
  """Sink for writing TFRecords files.

  For detailed TFRecord format description see:
    https://www.tensorflow.org/versions/r1.11/api_guides/python/python_io#TFRecords_Format_Details
  """

  def __init__(self, file_path_prefix, coder, file_name_suffix, num_shards,
               shard_name_template, compression_type):
    """Initialize a TFRecordSink. See WriteToTFRecord for details."""

    super(_TFRecordSink, self).__init__(
        file_path_prefix=file_path_prefix,
        coder=coder,
        file_name_suffix=file_name_suffix,
        num_shards=num_shards,
        shard_name_template=shard_name_template,
        mime_type='application/octet-stream',
        compression_type=compression_type)

  def write_encoded_record(self, file_handle, value):
    _TFRecordUtil.write_record(file_handle, value)

class WriteToTFRecord(PTransform):
  """Transform for writing to TFRecord sinks."""

  def __init__(self,
               file_path_prefix,
               coder=coders.BytesCoder(),
               file_name_suffix='',
               num_shards=0,
               shard_name_template=None,
               compression_type=CompressionTypes.AUTO,
               **kwargs):
    """Initialize WriteToTFRecord transform.

    Args:
      file_path_prefix: The file path to write to. The files written will begin
        with this prefix, followed by a shard identifier (see num_shards), and
        end in a common extension, if given by file_name_suffix.
      coder: Coder used to encode each record.
      file_name_suffix: Suffix for the files written.
      num_shards: The number of files (shards) used for output. If not set, the
        default value will be used.
      shard_name_template: A template string containing placeholders for
        the shard number and shard count. When constructing a filename for a
        particular shard number, the upper-case letters 'S' and 'N' are
        replaced with the 0-padded shard number and shard count respectively.
        This argument can be '' in which case it behaves as if num_shards was
        set to 1 and only one file will be generated. The default pattern used
        is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
      compression_type: Used to handle compressed output files. Typical value
          is CompressionTypes.AUTO, in which case the file_path's extension will
          be used to detect the compression.
      **kwargs: Optional args dictionary. These are passed through to parent
        constructor.

    Returns:
      A WriteToTFRecord transform object.
    """
    super(WriteToTFRecord, self).__init__(**kwargs)
    self._sink = _TFRecordSink(file_path_prefix, coder, file_name_suffix,
                               num_shards, shard_name_template,
                               compression_type)

  def expand(self, pcoll):
    return pcoll | Write(self._sink)
<|endoftext|>


<commit_before>import os
import sys
import regex
import regsub
import string
import getopt

def main():
	process(sys.stdin, sys.stdout)

dashes = regex.compile('^-+[ \t]*$')
equals = regex.compile('^=+[ \t]*$')
stars = regex.compile('^\*+[ \t]*$')
blank = regex.compile('^[ \t]*$')
indented = regex.compile('^\( *\t\|        \)[ \t]*[^ \t]')

def process(fi, fo):
	inverbatim = 0
	line = '\n'
	nextline = fi.readline()
	while nextline:
		prevline = line
		line = nextline
		nextline = fi.readline()
		fmt = None
		if dashes.match(nextline) >= 0:
			fmt = '\\subsection{%s}\n'
		elif equals.match(nextline) >= 0:
			fmt = '\\section{%s}\n'
		elif stars.match(nextline) >= 0:
			fmt = '\\chapter{%s}\n'
		if fmt:
			nextline = '\n'
			line =  fmt % string.strip(line)
			if '(' in line:
				line = regsub.gsub('[a-zA-Z0-9_]+()',
					  '{\\\\tt \\0}', line)
		elif inverbatim:
			if blank.match(line) >= 0 and \
				  indented.match(nextline) < 0:
				inverbatim = 0
				fo.write('\\end{verbatim}\n')
		else:
			if indented.match(line) >= 0 and \
				  blank.match(prevline) >= 0:
				inverbatim = 1
				fo.write('\\begin{verbatim}\n')
		if inverbatim:
			line = string.expandtabs(line, 4)
		elif not fmt and '(' in line:
			line = regsub.gsub('[a-zA-Z0-9_]+()',
				  '\\\\code{\\0}', line)
		fo.write(line)

main()
<commit_msg>No longer useful.<commit_after><|endoftext|>


<commit_before>from colander import (
    SchemaNode,
    Mapping,
    Sequence,
    SchemaType,
    String,
    Int,
    OneOf,
    Length,
    Regex,
    null,
)

class AlreadyRegisteredError(Exception):
    pass

class NotRegisteredError(Exception):
    pass

class UnknownFieldTypeError(NotRegisteredError):
    """Raised if schema contains a field with an unknown type."""
    pass

class TypeRegistry(object):
    """Registry containing all the types.

    This can be extended by third parties, and is always imported from
    daybed.schemas.
    """

    def __init__(self):
        self._registry = {}

    def register(self, name, klass):
        if name in self._registry:
            raise AlreadyRegisteredError('The type %s is already registered' %
                                         name)
        self._registry[name] = klass

    def unregister(self, name):
        if name not in self._registry:
            raise NotRegisteredError('The model %s is not registered' % name)
        del self._registry[name]

    def validation(self, typename, **options):
        try:
            nodetype = self._registry[typename]
        except KeyError:
            raise UnknownFieldTypeError('Type "%s" is unknown' % typename)
        return nodetype.validation(**options)

    def definition(self, typename, **options):
        try:
            nodetype = self._registry[typename]
        except KeyError:
            raise UnknownFieldTypeError('Type "%s" is unknown' % typename)
        return nodetype.definition(**options)

    @property
    def names(self):
        return self._registry.keys()

    def add(self, name):
        """Decorator to register new types"""
        def decorated(cls):
            self.register(name, cls)
            return cls
        return decorated

registry = TypeRegistry()

class TypeField(object):
    node = String

    @classmethod
    def definition(cls):
        schema = SchemaNode(Mapping())
        schema.add(SchemaNode(String(), name='name'))
        schema.add(SchemaNode(String(), name='description'))
        schema.add(SchemaNode(String(), name='type',
                              validator=OneOf(registry.names)))
        return schema

    @classmethod
    def validation(cls, **kwargs):
        keys = ['name', 'description', 'validator']
        options = dict(zip(keys, [kwargs.get(k) for k in keys]))
        return SchemaNode(cls.node(), **options)

@registry.add('int')
class IntField(TypeField):
    node = Int

@registry.add('string')
class StringField(TypeField):
    node = String

@registry.add('enum')
class EnumField(TypeField):
    node = String

    @classmethod
    def definition(cls):
        schema = super(EnumField, cls).definition()
        schema.add(SchemaNode(Sequence(), SchemaNode(String()),
                              name='choices', validator=Length(min=1)))
        return schema

    @classmethod
    def validation(cls, **kwargs):
        kwargs['validator'] = OneOf(kwargs['choices'])
        return super(EnumField, cls).validation(**kwargs)

@registry.add('regex')
class RegexField(TypeField):
    """Allows to validate a field with a python regular expression."""
    node = String

    @classmethod
    def definition(cls):
        schema = super(RegexField, cls).definition()
        schema.add(SchemaNode(String(), name='regex', validator=Length(min=1)))
        return schema

    @classmethod
    def validation(cls, **kwargs):
        kwargs['validator'] = Regex(kwargs['regex'])
        return super(RegexField, cls).validation(**kwargs)

class TypeFieldNode(SchemaType):
    def deserialize(self, node, cstruct=null):
        try:
            schema = registry.definition(cstruct.get('type'))
        except UnknownFieldTypeError:
            schema = TypeField.definition()
        schema.deserialize(cstruct)

class DefinitionValidator(SchemaNode):
    def __init__(self):
        super(DefinitionValidator, self).__init__(Mapping())
        self.add(SchemaNode(String(), name='title'))
        self.add(SchemaNode(String(), name='description'))
        self.add(SchemaNode(Sequence(), SchemaNode(TypeFieldNode()),
                            name='fields', validator=Length(min=1)))

class SchemaValidator(SchemaNode):
    def __init__(self, definition):
        super(SchemaValidator, self).__init__(Mapping())
        for field in definition['fields']:
            fieldtype = field.pop('type')
            self.add(registry.validation(fieldtype, **field))
<commit_msg>this is a class decorator.<commit_after>from colander import (
    SchemaNode,
    Mapping,
    Sequence,
    SchemaType,
    String,
    Int,
    OneOf,
    Length,
    Regex,
    null,
)

class AlreadyRegisteredError(Exception):
    pass

class NotRegisteredError(Exception):
    pass

class UnknownFieldTypeError(NotRegisteredError):
    """Raised if schema contains a field with an unknown type."""
    pass

class TypeRegistry(object):
    """Registry containing all the types.

    This can be extended by third parties, and is always imported from
    daybed.schemas.
    """

    def __init__(self):
        self._registry = {}

    def register(self, name, klass):
        if name in self._registry:
            raise AlreadyRegisteredError('The type %s is already registered' %
                                         name)
        self._registry[name] = klass

    def unregister(self, name):
        if name not in self._registry:
            raise NotRegisteredError('The model %s is not registered' % name)
        del self._registry[name]

    def validation(self, typename, **options):
        try:
            nodetype = self._registry[typename]
        except KeyError:
            raise UnknownFieldTypeError('Type "%s" is unknown' % typename)
        return nodetype.validation(**options)

    def definition(self, typename, **options):
        try:
            nodetype = self._registry[typename]
        except KeyError:
            raise UnknownFieldTypeError('Type "%s" is unknown' % typename)
        return nodetype.definition(**options)

    @property
    def names(self):
        return self._registry.keys()

    def add(self, name):
        """Class decorator, to register new types"""
        def decorated(cls):
            self.register(name, cls)
            return cls
        return decorated

registry = TypeRegistry()

class TypeField(object):
    node = String

    @classmethod
    def definition(cls):
        schema = SchemaNode(Mapping())
        schema.add(SchemaNode(String(), name='name'))
        schema.add(SchemaNode(String(), name='description'))
        schema.add(SchemaNode(String(), name='type',
                              validator=OneOf(registry.names)))
        return schema

    @classmethod
    def validation(cls, **kwargs):
        keys = ['name', 'description', 'validator']
        options = dict(zip(keys, [kwargs.get(k) for k in keys]))
        return SchemaNode(cls.node(), **options)

@registry.add('int')
class IntField(TypeField):
    node = Int

@registry.add('string')
class StringField(TypeField):
    node = String

@registry.add('enum')
class EnumField(TypeField):
    node = String

    @classmethod
    def definition(cls):
        schema = super(EnumField, cls).definition()
        schema.add(SchemaNode(Sequence(), SchemaNode(String()),
                              name='choices', validator=Length(min=1)))
        return schema

    @classmethod
    def validation(cls, **kwargs):
        kwargs['validator'] = OneOf(kwargs['choices'])
        return super(EnumField, cls).validation(**kwargs)

@registry.add('regex')
class RegexField(TypeField):
    """Allows to validate a field with a python regular expression."""
    node = String

    @classmethod
    def definition(cls):
        schema = super(RegexField, cls).definition()
        schema.add(SchemaNode(String(), name='regex', validator=Length(min=1)))
        return schema

    @classmethod
    def validation(cls, **kwargs):
        kwargs['validator'] = Regex(kwargs['regex'])
        return super(RegexField, cls).validation(**kwargs)

class TypeFieldNode(SchemaType):
    def deserialize(self, node, cstruct=null):
        try:
            schema = registry.definition(cstruct.get('type'))
        except UnknownFieldTypeError:
            schema = TypeField.definition()
        schema.deserialize(cstruct)

class DefinitionValidator(SchemaNode):
    def __init__(self):
        super(DefinitionValidator, self).__init__(Mapping())
        self.add(SchemaNode(String(), name='title'))
        self.add(SchemaNode(String(), name='description'))
        self.add(SchemaNode(Sequence(), SchemaNode(TypeFieldNode()),
                            name='fields', validator=Length(min=1)))

class SchemaValidator(SchemaNode):
    def __init__(self, definition):
        super(SchemaValidator, self).__init__(Mapping())
        for field in definition['fields']:
            fieldtype = field.pop('type')
            self.add(registry.validation(fieldtype, **field))
<|endoftext|>