<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import abc
import copy
import functools
import inspect

import jsonschema
import six
from six.moves import http_client
from webob import exc
import pecan
import traceback
from oslo_config import cfg

from st2common.constants.pack import DEFAULT_PACK_NAME
from st2common.util import mongoescape as util_mongodb
from st2common.util import schema as util_schema
from st2common.util.debugging import is_enabled as is_debugging_enabled
from st2common.util.jsonify import json_encode
from st2common import log as logging

__all__ = [
    'BaseAPI',

    'APIUIDMixin',

    'jsexpose'
]

LOG = logging.getLogger(__name__)

@six.add_metaclass(abc.ABCMeta)
class BaseAPI(object):
    schema = abc.abstractproperty

    def __init__(self, **kw):
        for key, value in kw.items():
            setattr(self, key, value)

    def __repr__(self):
        name = type(self).__name__
        attrs = ', '.join("'%s': %r" % item for item in six.iteritems(vars(self)))
        # The format here is so that eval can be applied.
        return "%s(**{%s})" % (name, attrs)

    def __str__(self):
        name = type(self).__name__
        attrs = ', '.join("%s=%r" % item for item in six.iteritems(vars(self)))

        return "%s[%s]" % (name, attrs)

    def __json__(self):
        return vars(self)

    def validate(self):
        """
        Perform validation and return cleaned object on success.

        Note: This method doesn't mutate this object in place, but it returns a new one.

        :return: Cleaned / validated object.
        """
        schema = getattr(self, 'schema', {})
        attributes = vars(self)

        cleaned = util_schema.validate(instance=attributes, schema=schema,
                                       cls=util_schema.CustomValidator, use_default=True,
                                       allow_default_none=True)

        return self.__class__(**cleaned)

    @classmethod
    def _from_model(cls, model, mask_secrets=False):
        doc = util_mongodb.unescape_chars(model.to_mongo())

        if '_id' in doc:
            doc['id'] = str(doc.pop('_id'))

        if mask_secrets and cfg.CONF.log.mask_secrets:
            doc = model.mask_secrets(value=doc)

        return doc

    @classmethod
    def from_model(cls, model, mask_secrets=False):
        """
        Create API model class instance for the provided DB model instance.

        :param model: DB model class instance.
        :type model: :class:`StormFoundationDB`

        :param mask_secrets: True to mask secrets in the resulting instance.
        :type mask_secrets: ``boolean``
        """
        doc = cls._from_model(model=model, mask_secrets=mask_secrets)
        attrs = {attr: value for attr, value in six.iteritems(doc) if value is not None}

        return cls(**attrs)

    @classmethod
    def to_model(cls, doc):
        """
        Create a model class instance for the provided MongoDB document.

        :param doc: MongoDB document.
        """
        raise NotImplementedError()

class APIUIDMixin(object):
    """"
    Mixin class for retrieving UID for API objects.
    """

    def get_uid(self):
        # TODO: This is not the most efficient approach - refactor this functionality into util
        # module and re-use it here and in the DB model
        resource_db = self.to_model(self)
        resource_uid = resource_db.get_uid()
        return resource_uid

    def get_pack_uid(self):
        # TODO: This is not the most efficient approach - refactor this functionality into util
        # module and re-use it here and in the DB model
        resource_db = self.to_model(self)
        pack_uid = resource_db.get_pack_uid()
        return pack_uid

def jsexpose(arg_types=None, body_cls=None, status_code=None, content_type='application/json'):
    """
    :param arg_types: A list of types for the function arguments.
    :type arg_types: ``list``

    :param body_cls: Request body class. If provided, this class will be used to create an instance
                     out of the request body.
    :type body_cls: :class:`object`

    :param status_code: Response status code.
    :type status_code: ``int``

    :param content_type: Response content type.
    :type content_type: ``str``
    """
    pecan_json_decorate = pecan.expose(
        content_type=content_type,
        generic=False)

    def decorate(f):
        @functools.wraps(f)
        def callfunction(*args, **kwargs):
            function_name = f.__name__
            args = list(args)
            types = copy.copy(arg_types)
            more = [args.pop(0)]

            def cast_value(value_type, value):
                if value_type == bool:
                    def cast_func(value):
                        return value.lower() in ['1', 'true']
                else:
                    cast_func = value_type

                result = cast_func(value)
                return result

            if types:
                argspec = inspect.getargspec(f)
                names = argspec.args[1:]

                for name in names:
                    try:
                        value = args.pop(0)
                        value = cast_value(value_type=types.pop(0), value=value)
                        more.append(value)
                    except IndexError:
                        try:
                            value = kwargs[name]
                            value = cast_value(value_type=types.pop(0), value=value)
                            kwargs[name] = value
                        except IndexError:
                            LOG.warning("Type definition for '%s' argument of '%s' "
                                        "is missing.", name, f.__name__)
                        except KeyError:
                            pass

            if body_cls:
                if pecan.request.body:
                    data = pecan.request.json
                else:
                    data = {}

                obj = body_cls(**data)
                try:
                    obj = obj.validate()
                except (jsonschema.ValidationError, ValueError) as e:
                    raise exc.HTTPBadRequest(detail=e.message,
                                             comment=traceback.format_exc())
                except Exception as e:
                    raise exc.HTTPInternalServerError(detail=e.message,
                                                      comment=traceback.format_exc())

                # Set default pack if one is not provided for resource create
                if function_name == 'post' and not hasattr(obj, 'pack'):
                    extra = {
                        'resource_api': obj,
                        'default_pack_name': DEFAULT_PACK_NAME
                    }
                    LOG.debug('Pack not provided in the body, setting a default pack name',
                              extra=extra)
                    setattr(obj, 'pack', DEFAULT_PACK_NAME)

                more.append(obj)

            args = tuple(more) + tuple(args)

            noop_codes = [http_client.NOT_IMPLEMENTED,
                          http_client.METHOD_NOT_ALLOWED,
                          http_client.FORBIDDEN]

            if status_code and status_code in noop_codes:
                pecan.response.status = status_code
                return json_encode(None)

            try:
                result = f(*args, **kwargs)
            except TypeError as e:
                message = str(e)
                # Invalid number of arguments passed to the function meaning invalid path was
                # requested
                # Note: The check is hacky, but it works for now.
                func_name = f.__name__
                pattern = '%s\(\) takes exactly \d+ arguments \(\d+ given\)' % (func_name)

                if re.search(pattern, message):
                    raise exc.HTTPNotFound()
                else:
                    raise e

            if status_code:
                pecan.response.status = status_code
            if content_type == 'application/json':
                if is_debugging_enabled():
                    indent = 4
                else:
                    indent = None
                return json_encode(result, indent=indent)
            else:
                return result

        pecan_json_decorate(callfunction)

        return callfunction

    return decorate
<commit_msg>Always handle keyword arguments first - this way arg_types works correctly, even if not all of the arguments are provided via the API.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import abc
import copy
import functools
import inspect

import jsonschema
import six
from six.moves import http_client
from webob import exc
import pecan
import traceback
from oslo_config import cfg

from st2common.constants.pack import DEFAULT_PACK_NAME
from st2common.util import mongoescape as util_mongodb
from st2common.util import schema as util_schema
from st2common.util.debugging import is_enabled as is_debugging_enabled
from st2common.util.jsonify import json_encode
from st2common import log as logging

__all__ = [
    'BaseAPI',

    'APIUIDMixin',

    'jsexpose'
]

LOG = logging.getLogger(__name__)

@six.add_metaclass(abc.ABCMeta)
class BaseAPI(object):
    schema = abc.abstractproperty

    def __init__(self, **kw):
        for key, value in kw.items():
            setattr(self, key, value)

    def __repr__(self):
        name = type(self).__name__
        attrs = ', '.join("'%s': %r" % item for item in six.iteritems(vars(self)))
        # The format here is so that eval can be applied.
        return "%s(**{%s})" % (name, attrs)

    def __str__(self):
        name = type(self).__name__
        attrs = ', '.join("%s=%r" % item for item in six.iteritems(vars(self)))

        return "%s[%s]" % (name, attrs)

    def __json__(self):
        return vars(self)

    def validate(self):
        """
        Perform validation and return cleaned object on success.

        Note: This method doesn't mutate this object in place, but it returns a new one.

        :return: Cleaned / validated object.
        """
        schema = getattr(self, 'schema', {})
        attributes = vars(self)

        cleaned = util_schema.validate(instance=attributes, schema=schema,
                                       cls=util_schema.CustomValidator, use_default=True,
                                       allow_default_none=True)

        return self.__class__(**cleaned)

    @classmethod
    def _from_model(cls, model, mask_secrets=False):
        doc = util_mongodb.unescape_chars(model.to_mongo())

        if '_id' in doc:
            doc['id'] = str(doc.pop('_id'))

        if mask_secrets and cfg.CONF.log.mask_secrets:
            doc = model.mask_secrets(value=doc)

        return doc

    @classmethod
    def from_model(cls, model, mask_secrets=False):
        """
        Create API model class instance for the provided DB model instance.

        :param model: DB model class instance.
        :type model: :class:`StormFoundationDB`

        :param mask_secrets: True to mask secrets in the resulting instance.
        :type mask_secrets: ``boolean``
        """
        doc = cls._from_model(model=model, mask_secrets=mask_secrets)
        attrs = {attr: value for attr, value in six.iteritems(doc) if value is not None}

        return cls(**attrs)

    @classmethod
    def to_model(cls, doc):
        """
        Create a model class instance for the provided MongoDB document.

        :param doc: MongoDB document.
        """
        raise NotImplementedError()

class APIUIDMixin(object):
    """"
    Mixin class for retrieving UID for API objects.
    """

    def get_uid(self):
        # TODO: This is not the most efficient approach - refactor this functionality into util
        # module and re-use it here and in the DB model
        resource_db = self.to_model(self)
        resource_uid = resource_db.get_uid()
        return resource_uid

    def get_pack_uid(self):
        # TODO: This is not the most efficient approach - refactor this functionality into util
        # module and re-use it here and in the DB model
        resource_db = self.to_model(self)
        pack_uid = resource_db.get_pack_uid()
        return pack_uid

def jsexpose(arg_types=None, body_cls=None, status_code=None, content_type='application/json'):
    """
    :param arg_types: A list of types for the function arguments.
    :type arg_types: ``list``

    :param body_cls: Request body class. If provided, this class will be used to create an instance
                     out of the request body.
    :type body_cls: :class:`object`

    :param status_code: Response status code.
    :type status_code: ``int``

    :param content_type: Response content type.
    :type content_type: ``str``
    """
    pecan_json_decorate = pecan.expose(
        content_type=content_type,
        generic=False)

    def decorate(f):
        @functools.wraps(f)
        def callfunction(*args, **kwargs):
            function_name = f.__name__
            args = list(args)
            types = copy.copy(arg_types)
            more = [args.pop(0)]

            def cast_value(value_type, value):
                if value_type == bool:
                    def cast_func(value):
                        return value.lower() in ['1', 'true']
                else:
                    cast_func = value_type

                result = cast_func(value)
                return result

            if types:
                argspec = inspect.getargspec(f)
                names = argspec.args[1:]

                for index, name in enumerate(names):
                    # 1. Try kwargs first
                    if name in kwargs:
                        try:
                            value = kwargs[name]
                            value_type = types[index]
                            value = cast_value(value_type=value_type, value=value)
                            kwargs[name] = value
                        except IndexError:
                            LOG.warning("Type definition for '%s' argument of '%s' "
                                        "is missing.", name, f.__name__)

                        continue

                    # 2. Try positional args
                    try:
                        value = args.pop(0)
                        value_type = types[index]
                        value = cast_value(value_type=value_type, value=value)
                        more.append(value)
                    except IndexError:
                        LOG.warning("Type definition for '%s' argument of '%s' "
                                    "is missing.", name, f.__name__)

            if body_cls:
                if pecan.request.body:
                    data = pecan.request.json
                else:
                    data = {}

                obj = body_cls(**data)
                try:
                    obj = obj.validate()
                except (jsonschema.ValidationError, ValueError) as e:
                    raise exc.HTTPBadRequest(detail=e.message,
                                             comment=traceback.format_exc())
                except Exception as e:
                    raise exc.HTTPInternalServerError(detail=e.message,
                                                      comment=traceback.format_exc())

                # Set default pack if one is not provided for resource create
                if function_name == 'post' and not hasattr(obj, 'pack'):
                    extra = {
                        'resource_api': obj,
                        'default_pack_name': DEFAULT_PACK_NAME
                    }
                    LOG.debug('Pack not provided in the body, setting a default pack name',
                              extra=extra)
                    setattr(obj, 'pack', DEFAULT_PACK_NAME)

                more.append(obj)

            args = tuple(more) + tuple(args)

            noop_codes = [http_client.NOT_IMPLEMENTED,
                          http_client.METHOD_NOT_ALLOWED,
                          http_client.FORBIDDEN]

            if status_code and status_code in noop_codes:
                pecan.response.status = status_code
                return json_encode(None)

            try:
                result = f(*args, **kwargs)
            except TypeError as e:
                message = str(e)
                # Invalid number of arguments passed to the function meaning invalid path was
                # requested
                # Note: The check is hacky, but it works for now.
                func_name = f.__name__
                pattern = '%s\(\) takes exactly \d+ arguments \(\d+ given\)' % (func_name)

                if re.search(pattern, message):
                    raise exc.HTTPNotFound()
                else:
                    raise e

            if status_code:
                pecan.response.status = status_code
            if content_type == 'application/json':
                if is_debugging_enabled():
                    indent = 4
                else:
                    indent = None
                return json_encode(result, indent=indent)
            else:
                return result

        pecan_json_decorate(callfunction)

        return callfunction

    return decorate
<|endoftext|>


<commit_before>import operator

import pytest

from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies
from plenum.test.primary_selection.helper import \
    check_rank_consistent_across_each_node
from plenum.test.view_change.helper import ensure_view_change
from stp_core.loop.eventually import eventually
from plenum.common.util import getNoInstances
from plenum.server.replica import Replica
from plenum.test import waits
from plenum.test.test_node import checkProtocolInstanceSetup, getPrimaryReplica, ensureElectionsDone

# noinspection PyUnresolvedReferences
from plenum.test.view_change.conftest import viewNo

# noinspection PyUnresolvedReferences
from plenum.test.view_change.test_view_change import viewChangeDone
# noinspection PyUnresolvedReferences
from plenum.test.conftest import looper, client1, wallet1, clientAndWallet1

nodeCount = 7

@pytest.fixture()
def primaryReplicas(nodeSet, up):
    instanceCount = getNoInstances(nodeCount)
    return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]

# noinspection PyIncorrectDocstring
def testPrimarySelectionAfterPoolReady(looper, nodeSet, ready, wallet1, client1):
    """
    Once the pool is ready(node has connected to at least 3 other nodes),
    appropriate primary replicas should be selected.
    """
    def checkPrimaryPlacement():
        # Node names sorted by rank
        sortedNodeNames = sorted(nodeSet.nodes.values(),
                                 key=operator.attrgetter("rank"))

        for idx, node in enumerate(sortedNodeNames):
            # For instance 0, the primary replica should be on the node with rank 0
            if idx == 0:
                Replica.generateName(sortedNodeNames[idx], 0)
                assert node.replicas[0].isPrimary
                assert not node.replicas[1].isPrimary
                assert not node.replicas[2].isPrimary

            # For instance 1, the primary replica should be on the node with rank 1
            if idx == 1:
                Replica.generateName(sortedNodeNames[idx], 1)
                assert not node.replicas[0].isPrimary
                assert node.replicas[1].isPrimary
                assert not node.replicas[2].isPrimary

            # For instance 2, the primary replica should be on the node with rank 2
            if idx == 2:
                Replica.generateName(sortedNodeNames[idx], 2)
                assert not node.replicas[0].isPrimary
                assert not node.replicas[1].isPrimary
                assert node.replicas[2].isPrimary

    check_rank_consistent_across_each_node(nodeSet)
    # Check if the primary is on the correct node
    timeout = waits.expectedPoolElectionTimeout(len(nodeSet))
    looper.run(eventually(checkPrimaryPlacement, retryWait=1, timeout=timeout))
    # Check if every protocol instance has one and only one primary and any node
    #  has no more than one primary
    checkProtocolInstanceSetup(looper, nodeSet, retryWait=1)
    sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5)

@pytest.fixture(scope='module')
def catchup_complete_count(nodeSet):
    return {n.name: n.spylog.count(n.allLedgersCaughtUp) for n in nodeSet}

@pytest.fixture(scope='module')
def view_change_done(looper, nodeSet):
    ensure_view_change(looper, nodeSet)
    ensureElectionsDone(looper=looper, nodes=nodeSet)

# noinspection PyIncorrectDocstring
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas,
                                        catchup_complete_count, view_change_done):
    """
    Test that primary replica of a protocol instance shifts to a new node after
    a view change.
    """
    # TODO: This test can fail due to view change.

    for n in nodeSet:
        assert n.spylog.count(n.allLedgersCaughtUp) > catchup_complete_count[n.name]

    # Primary replicas before view change
    prBeforeVC = primaryReplicas

    # Primary replicas after view change
    instanceCount = getNoInstances(nodeCount)
    prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]

    # Primary replicas have moved to the next node
    for br, ar in zip(prBeforeVC, prAfterVC):
        assert ar.node.rank - br.node.rank == 1

    check_rank_consistent_across_each_node(nodeSet)
    checkProtocolInstanceSetup(looper, nodeSet, retryWait=1)
<commit_msg>revert a fix for the tests<commit_after>import operator

import pytest

from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies
from plenum.test.primary_selection.helper import \
    check_rank_consistent_across_each_node
from plenum.test.view_change.helper import ensure_view_change
from stp_core.loop.eventually import eventually
from plenum.common.util import getNoInstances
from plenum.server.replica import Replica
from plenum.test import waits
from plenum.test.test_node import checkProtocolInstanceSetup, getPrimaryReplica, ensureElectionsDone

# noinspection PyUnresolvedReferences
from plenum.test.view_change.conftest import viewNo

# noinspection PyUnresolvedReferences
from plenum.test.conftest import looper, client1, wallet1, clientAndWallet1

nodeCount = 7

@pytest.fixture()
def primaryReplicas(nodeSet, up):
    instanceCount = getNoInstances(nodeCount)
    return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]

# noinspection PyIncorrectDocstring
def testPrimarySelectionAfterPoolReady(looper, nodeSet, ready, wallet1, client1):
    """
    Once the pool is ready(node has connected to at least 3 other nodes),
    appropriate primary replicas should be selected.
    """
    def checkPrimaryPlacement():
        # Node names sorted by rank
        sortedNodeNames = sorted(nodeSet.nodes.values(),
                                 key=operator.attrgetter("rank"))

        for idx, node in enumerate(sortedNodeNames):
            # For instance 0, the primary replica should be on the node with rank 0
            if idx == 0:
                Replica.generateName(sortedNodeNames[idx], 0)
                assert node.replicas[0].isPrimary
                assert not node.replicas[1].isPrimary
                assert not node.replicas[2].isPrimary

            # For instance 1, the primary replica should be on the node with rank 1
            if idx == 1:
                Replica.generateName(sortedNodeNames[idx], 1)
                assert not node.replicas[0].isPrimary
                assert node.replicas[1].isPrimary
                assert not node.replicas[2].isPrimary

            # For instance 2, the primary replica should be on the node with rank 2
            if idx == 2:
                Replica.generateName(sortedNodeNames[idx], 2)
                assert not node.replicas[0].isPrimary
                assert not node.replicas[1].isPrimary
                assert node.replicas[2].isPrimary

    check_rank_consistent_across_each_node(nodeSet)
    # Check if the primary is on the correct node
    timeout = waits.expectedPoolElectionTimeout(len(nodeSet))
    looper.run(eventually(checkPrimaryPlacement, retryWait=1, timeout=timeout))
    # Check if every protocol instance has one and only one primary and any node
    #  has no more than one primary
    checkProtocolInstanceSetup(looper, nodeSet, retryWait=1)
    sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5)

@pytest.fixture(scope='module')
def catchup_complete_count(nodeSet):
    return {n.name: n.spylog.count(n.allLedgersCaughtUp) for n in nodeSet}

@pytest.fixture(scope='module')
def view_change_done(looper, nodeSet):
    ensure_view_change(looper, nodeSet)
    ensureElectionsDone(looper=looper, nodes=nodeSet)

@pytest.fixture(scope='module')
def view_change_done(looper, nodeSet):
    ensure_view_change(looper, nodeSet)
    ensureElectionsDone(looper=looper, nodes=nodeSet)

# noinspection PyIncorrectDocstring
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas,
                                        catchup_complete_count, view_change_done):
    """
    Test that primary replica of a protocol instance shifts to a new node after
    a view change.
    """
    # TODO: This test can fail due to view change.

    for n in nodeSet:
        assert n.spylog.count(n.allLedgersCaughtUp) > catchup_complete_count[n.name]

    # Primary replicas before view change
    prBeforeVC = primaryReplicas

    # Primary replicas after view change
    instanceCount = getNoInstances(nodeCount)
    prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]

    # Primary replicas have moved to the next node
    for br, ar in zip(prBeforeVC, prAfterVC):
        assert ar.node.rank - br.node.rank == 1

    check_rank_consistent_across_each_node(nodeSet)
    checkProtocolInstanceSetup(looper, nodeSet, retryWait=1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2016,  Digital Reasoning
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# pylint: disable=abstract-method

import logging
import string

from django.conf import settings
from django.db import transaction
from rest_framework import serializers

from stackdio.core.serializers import (
    StackdioHyperlinkedModelSerializer,
    StackdioParentHyperlinkedModelSerializer,
    StackdioLabelSerializer,
    StackdioLiteralLabelsSerializer,
)
from stackdio.core.utils import recursive_update, recursively_sort_dict
from stackdio.core.validators import PropertiesValidator
from stackdio.api.cloud.models import CloudImage, CloudInstanceSize, CloudZone, Snapshot
from stackdio.api.formulas.serializers import FormulaVersionSerializer, FormulaComponentSerializer
from stackdio.api.formulas.validators import validate_formula_components
from . import models, validators

logger = logging.getLogger(__name__)

class BlueprintPropertiesSerializer(serializers.Serializer):
    def to_representation(self, obj):
        ret = {}
        if obj is not None:
            # Make it work two different ways.. ooooh
            if isinstance(obj, models.Blueprint):
                ret = obj.properties
            else:
                ret = obj
        return recursively_sort_dict(ret)

    def to_internal_value(self, data):
        return data

    def validate(self, attrs):
        PropertiesValidator().validate(attrs)
        return attrs

    def update(self, instance, validated_data):
        blueprint = instance
        if self.partial:
            # This is a PATCH, so properly merge in the old data
            old_properties = blueprint.properties
            blueprint.properties = recursive_update(old_properties, validated_data)
        else:
            # This is a PUT, so just add the data directly
            blueprint.properties = validated_data

        # Be sure to persist the data
        blueprint.save()
        return blueprint

class BlueprintAccessRuleSerializer(serializers.ModelSerializer):
    from_port = serializers.IntegerField(min_value=0, max_value=65535)
    to_port = serializers.IntegerField(min_value=0, max_value=65535)

    class Meta:
        model = models.BlueprintAccessRule
        fields = (
            'protocol',
            'from_port',
            'to_port',
            'rule',
        )

    def validate(self, attrs):
        from_port = attrs['from_port']
        to_port = attrs['to_port']

        if from_port > to_port:
            err_msg = '`from_port` ({0}) must be less than `to_port` ({1})'.format(from_port,
                                                                                to_port)
            raise serializers.ValidationError({
                'from_port': [err_msg],
                'to_port': [err_msg],
            })

        return attrs

class BlueprintVolumeSerializer(serializers.ModelSerializer):
    snapshot = serializers.SlugRelatedField(slug_field='slug', queryset=Snapshot.objects.all())

    class Meta:
        model = models.BlueprintVolume
        fields = (
            'device',
            'mount_point',
            'snapshot',
        )

class BlueprintHostDefinitionSerializer(StackdioParentHyperlinkedModelSerializer):
    formula_components = FormulaComponentSerializer(many=True)
    access_rules = BlueprintAccessRuleSerializer(many=True, required=False)
    volumes = BlueprintVolumeSerializer(many=True, required=False)

    size = serializers.SlugRelatedField(slug_field='instance_id',
                                        queryset=CloudInstanceSize.objects.all())
    zone = serializers.SlugRelatedField(slug_field='title', required=False, allow_null=True,
                                        queryset=CloudZone.objects.all())
    cloud_image = serializers.SlugRelatedField(slug_field='slug',
                                               queryset=CloudImage.objects.all())

    class Meta:
        model = models.BlueprintHostDefinition
        model_name = 'blueprint-host-definition'
        parent_attr = 'blueprint'
        fields = (
            'id',
            'url',
            'title',
            'description',
            'cloud_image',
            'count',
            'hostname_template',
            'size',
            'zone',
            'subnet_id',
            'formula_components',
            'access_rules',
            'volumes',
            'spot_price',
        )

        extra_kwargs = {
            'spot_price': {'min_value': 0.0},
            'subnet_id': {'allow_null': True},
            'hostname_template': {'validators': [validators.BlueprintHostnameTemplateValidator()]},
        }

    def validate(self, attrs):
        hostname_template = attrs.get('hostname_template')

        errors = {}

        # Only validate the hostname template if we have one
        if hostname_template is not None:
            count = attrs.get('count')
            if count is None:
                if not self.instance:
                    raise ValueError('`count` not found and the serializer doesn\'t '
                                     'have an instance attribute')
                # Grab the count from the instance
                count = self.instance.count

            # Validate hostname template
            formatter = string.Formatter()
            template_vars = [x[1] for x in formatter.parse(hostname_template) if x[1]]

            if count > 1 and 'index' not in template_vars:
                err_msg = '`hostname_template` must contain "{index}" when `count` > 1'
                errors.setdefault('hostname_template', []).append(err_msg)
                errors.setdefault('count', []).append(err_msg)

        # Validate zone / subnet id
        image = attrs.get('cloud_image')
        if image is None:
            if not self.instance:
                raise ValueError('`cloud_image` not found and the serializer doesn\'t '
                                 'have an instance attribute')
            # Grab the image from the instance
            image = self.instance.cloud_image

        account = image.account

        if account.vpc_enabled:
            # If we're in a vpc, we need a VALID subnet id
            subnet_id = attrs.get('subnet_id')
            if subnet_id is None:
                err_msg = 'This is a required field for a image in a vpc enabled account.'
                errors.setdefault('subnet_id', []).append(err_msg)

            driver = image.get_driver()
            subnets = driver.get_vpc_subnets([subnet_id])
            if subnets is None:
                err_msg = '"{0}" is not a valid subnet id.'.format(subnet_id)
                errors.setdefault('subnet_id', []).append(err_msg)
        else:
            # if we're not in a VPC, we only need a zone
            zone = attrs.get('zone')
            if zone is None:
                err_msg = 'This is a required field for a provile in a non-vpc enabled account.'
                errors.setdefault('zone', []).append(err_msg)

        if errors:
            raise serializers.ValidationError(errors)

        return attrs

    def create(self, validated_data):
        formula_components = validated_data.pop('formula_components')
        access_rules = validated_data.pop('access_rules', [])
        volumes = validated_data.pop('volumes', [])

        host_definition = super(BlueprintHostDefinitionSerializer, self).create(validated_data)

        # Create the formula components
        formula_component_field = self.fields['formula_components']
        # Add in the host definition to all the formula components
        for component in formula_components:
            component['content_object'] = host_definition
        formula_component_field.create(formula_components)

        # Create the access rules
        access_rule_field = self.fields['access_rules']
        # Add in the host definition to all the access rules
        for access_rule in access_rules:
            access_rule['host'] = host_definition
        access_rule_field.create(access_rules)

        # Create the volumes
        volume_field = self.fields['volumes']
        # Add in the host definition to all the volumes
        for volume in volumes:
            volume['host'] = host_definition
        volume_field.create(volumes)

        return host_definition

class BlueprintSerializer(StackdioHyperlinkedModelSerializer):
    stack_count = serializers.ReadOnlyField(source='stacks.count')
    label_list = StackdioLiteralLabelsSerializer(read_only=True, many=True, source='labels')

    properties = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-properties')
    host_definitions = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-host-definition-list',
        lookup_url_kwarg='parent_pk')
    formula_versions = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-formula-versions',
        lookup_url_kwarg='parent_pk')
    labels = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-label-list',
        lookup_url_kwarg='parent_pk')
    export = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-export')
    user_permissions = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-object-user-permissions-list',
        lookup_url_kwarg='parent_pk')
    group_permissions = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-object-group-permissions-list',
        lookup_url_kwarg='parent_pk')

    class Meta:
        model = models.Blueprint
        fields = (
            'id',
            'url',
            'title',
            'description',
            'create_users',
            'stack_count',
            'label_list',
            'properties',
            'host_definitions',
            'formula_versions',
            'labels',
            'export',
            'user_permissions',
            'group_permissions',
        )

        extra_kwargs = {
            'create_users': {
                'default': serializers.CreateOnlyDefault(settings.STACKDIO_CONFIG.create_ssh_users)
            },
        }

class FullBlueprintSerializer(BlueprintSerializer):
    properties = BlueprintPropertiesSerializer(required=False)
    host_definitions = BlueprintHostDefinitionSerializer(many=True)
    formula_versions = FormulaVersionSerializer(many=True, required=False)
    labels = StackdioLiteralLabelsSerializer(many=True, required=False)

    def validate(self, attrs):
        host_definitions = attrs['host_definitions']
        formula_versions = attrs.get('formula_versions', [])
        if len(host_definitions) < 1:
            raise serializers.ValidationError({
                'host_definitions': ['You must supply at least 1 host definition.']
            })

        errors = {}

        # Make sure the titles don't conflict
        titles = []
        hostnames = []
        for host_definition in host_definitions:
            title = host_definition['title']
            hostname = host_definition['hostname_template']
            if title in titles:
                err_msg = 'Duplicate title: {0}'.format(title)
                errors.setdefault('host_definitions', []).append(err_msg)
            if hostname in hostnames:
                err_msg = 'Duplicate hostname template: {0}'.format(hostname)
                errors.setdefault('host_definitions', []).append(err_msg)
            titles.append(title)
            hostnames.append(hostname)

        # Validate component ordering, and ensure that component ordering matches
        # across host definitions
        order_set = set()
        component_map = {}

        # Grab all the orders from each component on each host_definition
        for host_definition in host_definitions:
            formula_components = host_definition['formula_components']

            # Validate the components (i.e. make sure they exist in the formula on the
            # specified version)
            formula_components = validate_formula_components(formula_components, formula_versions)

            for component in formula_components:
                # Add the order to the order set
                order_set.add(component['order'])
                # Also add the order to the component map
                component_map.setdefault(component['sls_path'], set()).add(component['order'])

        for component, orders in component_map.items():
            if len(orders) > 1:
                err_msg = 'Formula component "{0}" has inconsistent orders across host definitions.'
                errors.setdefault('host_definitions', []).append(err_msg.format(component))

        # Every number in the range [0, MAX_ORDER] should be represented
        if sorted(order_set) != range(len(order_set)):
            err_msg = ('Ordering is zero-based, may have duplicates across hosts, but '
                       'can not have any gaps in the order. '
                       'Your de-duplicated order: {0}'.format(sorted(order_set)))
            errors.setdefault('host_definitions', []).append(err_msg)

        if errors:
            raise serializers.ValidationError(errors)

        return attrs

    def create(self, validated_data):
        # Pull out the nested stuff
        host_definitions = validated_data.pop('host_definitions')
        properties = validated_data.pop('properties', {})
        formula_versions = validated_data.pop('formula_versions', [])
        labels = validated_data.pop('labels', [])

        with transaction.atomic(using=models.Blueprint.objects.db):
            # Create the blueprint
            blueprint = super(FullBlueprintSerializer, self).create(validated_data)

            # Set the properties
            blueprint.properties = properties

            # Create the host definitions
            host_definition_field = self.fields['host_definitions']
            # Add in the blueprint to all the host defs
            for host_definition in host_definitions:
                host_definition['blueprint'] = blueprint
            host_definition_field.create(host_definitions)

            # Create the formula versions
            formula_version_field = self.fields['formula_versions']
            # Add in the blueprint to all the formula versions
            for formula_version in formula_versions:
                formula_version['content_object'] = blueprint
            formula_version_field.create(formula_versions)

            # Create the labels
            label_field = self.fields['labels']
            # Add in the blueprint to all the labels
            for label in labels:
                label['content_object'] = blueprint
            label_field.create(labels)

        # Add the other fields back in for deserialization
        validated_data['properties'] = properties
        validated_data['host_definitions'] = host_definitions
        validated_data['formula_versions'] = formula_versions
        validated_data['labels'] = labels

        return blueprint

class BlueprintExportSerializer(FullBlueprintSerializer):
    class Meta:
        model = models.Blueprint
        fields = (
            'title',
            'description',
            'create_users',
            'properties',
            'labels',
            'host_definitions',
            'formula_versions',
        )

class BlueprintLabelSerializer(StackdioLabelSerializer):

    class Meta(StackdioLabelSerializer.Meta):
        app_label = 'blueprints'
        model_name = 'blueprint-label'
<commit_msg>Added create only fields to the host definition<commit_after># -*- coding: utf-8 -*-

# Copyright 2016,  Digital Reasoning
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# pylint: disable=abstract-method

import logging
import string

from django.conf import settings
from django.db import transaction
from rest_framework import serializers
from stackdio.api.cloud.models import CloudImage, CloudInstanceSize, CloudZone, Snapshot
from stackdio.api.formulas.serializers import FormulaVersionSerializer, FormulaComponentSerializer
from stackdio.api.formulas.validators import validate_formula_components
from stackdio.core.mixins import CreateOnlyFieldsMixin
from stackdio.core.serializers import (
    StackdioHyperlinkedModelSerializer,
    StackdioParentHyperlinkedModelSerializer,
    StackdioLabelSerializer,
    StackdioLiteralLabelsSerializer,
)
from stackdio.core.utils import recursive_update, recursively_sort_dict
from stackdio.core.validators import PropertiesValidator

from . import models, validators

logger = logging.getLogger(__name__)

class BlueprintPropertiesSerializer(serializers.Serializer):
    def to_representation(self, obj):
        ret = {}
        if obj is not None:
            # Make it work two different ways.. ooooh
            if isinstance(obj, models.Blueprint):
                ret = obj.properties
            else:
                ret = obj
        return recursively_sort_dict(ret)

    def to_internal_value(self, data):
        return data

    def validate(self, attrs):
        PropertiesValidator().validate(attrs)
        return attrs

    def update(self, instance, validated_data):
        blueprint = instance
        if self.partial:
            # This is a PATCH, so properly merge in the old data
            old_properties = blueprint.properties
            blueprint.properties = recursive_update(old_properties, validated_data)
        else:
            # This is a PUT, so just add the data directly
            blueprint.properties = validated_data

        # Be sure to persist the data
        blueprint.save()
        return blueprint

class BlueprintAccessRuleSerializer(serializers.ModelSerializer):
    from_port = serializers.IntegerField(min_value=0, max_value=65535)
    to_port = serializers.IntegerField(min_value=0, max_value=65535)

    class Meta:
        model = models.BlueprintAccessRule
        fields = (
            'protocol',
            'from_port',
            'to_port',
            'rule',
        )

    def validate(self, attrs):
        from_port = attrs['from_port']
        to_port = attrs['to_port']

        if from_port > to_port:
            err_msg = '`from_port` ({0}) must be less than `to_port` ({1})'.format(from_port,
                                                                                to_port)
            raise serializers.ValidationError({
                'from_port': [err_msg],
                'to_port': [err_msg],
            })

        return attrs

class BlueprintVolumeSerializer(serializers.ModelSerializer):
    snapshot = serializers.SlugRelatedField(slug_field='slug', queryset=Snapshot.objects.all())

    class Meta:
        model = models.BlueprintVolume
        fields = (
            'device',
            'mount_point',
            'snapshot',
        )

class BlueprintHostDefinitionSerializer(CreateOnlyFieldsMixin,
                                        StackdioParentHyperlinkedModelSerializer):
    formula_components = FormulaComponentSerializer(many=True)
    access_rules = BlueprintAccessRuleSerializer(many=True, required=False)
    volumes = BlueprintVolumeSerializer(many=True, required=False)

    size = serializers.SlugRelatedField(slug_field='instance_id',
                                        queryset=CloudInstanceSize.objects.all())
    zone = serializers.SlugRelatedField(slug_field='title', required=False, allow_null=True,
                                        queryset=CloudZone.objects.all())
    cloud_image = serializers.SlugRelatedField(slug_field='slug',
                                               queryset=CloudImage.objects.all())

    class Meta:
        model = models.BlueprintHostDefinition
        model_name = 'blueprint-host-definition'
        parent_attr = 'blueprint'
        fields = (
            'id',
            'url',
            'title',
            'description',
            'cloud_image',
            'count',
            'hostname_template',
            'size',
            'zone',
            'subnet_id',
            'spot_price',
            'formula_components',
            'access_rules',
            'volumes',
        )

        create_only_fields = (
            'cloud_image',
            'size',
            'formula_components',
            'access_rules',
            'volumes',
        )

        extra_kwargs = {
            'spot_price': {'min_value': 0.0},
            'subnet_id': {'allow_null': True},
            'hostname_template': {'validators': [validators.BlueprintHostnameTemplateValidator()]},
        }

    def validate(self, attrs):
        hostname_template = attrs.get('hostname_template')

        errors = {}

        # Only validate the hostname template if we have one
        if hostname_template is not None:
            count = attrs.get('count')
            if count is None:
                if not self.instance:
                    raise ValueError('`count` not found and the serializer doesn\'t '
                                     'have an instance attribute')
                # Grab the count from the instance
                count = self.instance.count

            # Validate hostname template
            formatter = string.Formatter()
            template_vars = [x[1] for x in formatter.parse(hostname_template) if x[1]]

            if count > 1 and 'index' not in template_vars:
                err_msg = '`hostname_template` must contain "{index}" when `count` > 1'
                errors.setdefault('hostname_template', []).append(err_msg)
                errors.setdefault('count', []).append(err_msg)

        # Validate zone / subnet id
        image = attrs.get('cloud_image')
        if image is None:
            if not self.instance:
                raise ValueError('`cloud_image` not found and the serializer doesn\'t '
                                 'have an instance attribute')
            # Grab the image from the instance
            image = self.instance.cloud_image

        account = image.account

        if account.vpc_enabled:
            # If we're in a vpc, we need a VALID subnet id
            subnet_id = attrs.get('subnet_id')
            if subnet_id is None:
                err_msg = 'This is a required field for a image in a vpc enabled account.'
                errors.setdefault('subnet_id', []).append(err_msg)

            driver = image.get_driver()
            subnets = driver.get_vpc_subnets([subnet_id])
            if subnets is None:
                err_msg = '"{0}" is not a valid subnet id.'.format(subnet_id)
                errors.setdefault('subnet_id', []).append(err_msg)
        else:
            # if we're not in a VPC, we only need a zone
            zone = attrs.get('zone')
            if zone is None:
                err_msg = 'This is a required field for a provile in a non-vpc enabled account.'
                errors.setdefault('zone', []).append(err_msg)

        if errors:
            raise serializers.ValidationError(errors)

        return attrs

    def create(self, validated_data):
        formula_components = validated_data.pop('formula_components')
        access_rules = validated_data.pop('access_rules', [])
        volumes = validated_data.pop('volumes', [])

        host_definition = super(BlueprintHostDefinitionSerializer, self).create(validated_data)

        # Create the formula components
        formula_component_field = self.fields['formula_components']
        # Add in the host definition to all the formula components
        for component in formula_components:
            component['content_object'] = host_definition
        formula_component_field.create(formula_components)

        # Create the access rules
        access_rule_field = self.fields['access_rules']
        # Add in the host definition to all the access rules
        for access_rule in access_rules:
            access_rule['host'] = host_definition
        access_rule_field.create(access_rules)

        # Create the volumes
        volume_field = self.fields['volumes']
        # Add in the host definition to all the volumes
        for volume in volumes:
            volume['host'] = host_definition
        volume_field.create(volumes)

        return host_definition

class BlueprintSerializer(StackdioHyperlinkedModelSerializer):
    stack_count = serializers.ReadOnlyField(source='stacks.count', read_only=True)
    label_list = StackdioLiteralLabelsSerializer(read_only=True, many=True, source='labels')

    properties = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-properties')
    host_definitions = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-host-definition-list',
        lookup_url_kwarg='parent_pk')
    formula_versions = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-formula-versions',
        lookup_url_kwarg='parent_pk')
    labels = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-label-list',
        lookup_url_kwarg='parent_pk')
    export = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-export')
    user_permissions = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-object-user-permissions-list',
        lookup_url_kwarg='parent_pk')
    group_permissions = serializers.HyperlinkedIdentityField(
        view_name='api:blueprints:blueprint-object-group-permissions-list',
        lookup_url_kwarg='parent_pk')

    class Meta:
        model = models.Blueprint
        fields = (
            'id',
            'url',
            'title',
            'description',
            'create_users',
            'stack_count',
            'label_list',
            'properties',
            'host_definitions',
            'formula_versions',
            'labels',
            'export',
            'user_permissions',
            'group_permissions',
        )

        extra_kwargs = {
            'create_users': {
                'default': serializers.CreateOnlyDefault(settings.STACKDIO_CONFIG.create_ssh_users)
            },
        }

class FullBlueprintSerializer(BlueprintSerializer):
    properties = BlueprintPropertiesSerializer(required=False)
    host_definitions = BlueprintHostDefinitionSerializer(many=True)
    formula_versions = FormulaVersionSerializer(many=True, required=False)
    labels = StackdioLiteralLabelsSerializer(many=True, required=False)

    def validate(self, attrs):
        host_definitions = attrs['host_definitions']
        formula_versions = attrs.get('formula_versions', [])
        if len(host_definitions) < 1:
            raise serializers.ValidationError({
                'host_definitions': ['You must supply at least 1 host definition.']
            })

        errors = {}

        # Make sure the titles don't conflict
        titles = []
        hostnames = []
        for host_definition in host_definitions:
            title = host_definition['title']
            hostname = host_definition['hostname_template']
            if title in titles:
                err_msg = 'Duplicate title: {0}'.format(title)
                errors.setdefault('host_definitions', []).append(err_msg)
            if hostname in hostnames:
                err_msg = 'Duplicate hostname template: {0}'.format(hostname)
                errors.setdefault('host_definitions', []).append(err_msg)
            titles.append(title)
            hostnames.append(hostname)

        # Validate component ordering, and ensure that component ordering matches
        # across host definitions
        order_set = set()
        component_map = {}

        # Grab all the orders from each component on each host_definition
        for host_definition in host_definitions:
            formula_components = host_definition['formula_components']

            # Validate the components (i.e. make sure they exist in the formula on the
            # specified version)
            formula_components = validate_formula_components(formula_components, formula_versions)

            for component in formula_components:
                # Add the order to the order set
                order_set.add(component['order'])
                # Also add the order to the component map
                component_map.setdefault(component['sls_path'], set()).add(component['order'])

        for component, orders in component_map.items():
            if len(orders) > 1:
                err_msg = 'Formula component "{0}" has inconsistent orders across host definitions.'
                errors.setdefault('host_definitions', []).append(err_msg.format(component))

        # Every number in the range [0, MAX_ORDER] should be represented
        if sorted(order_set) != range(len(order_set)):
            err_msg = ('Ordering is zero-based, may have duplicates across hosts, but '
                       'can not have any gaps in the order. '
                       'Your de-duplicated order: {0}'.format(sorted(order_set)))
            errors.setdefault('host_definitions', []).append(err_msg)

        if errors:
            raise serializers.ValidationError(errors)

        return attrs

    def create(self, validated_data):
        # Pull out the nested stuff
        host_definitions = validated_data.pop('host_definitions')
        properties = validated_data.pop('properties', {})
        formula_versions = validated_data.pop('formula_versions', [])
        labels = validated_data.pop('labels', [])

        with transaction.atomic(using=models.Blueprint.objects.db):
            # Create the blueprint
            blueprint = super(FullBlueprintSerializer, self).create(validated_data)

            # Set the properties
            blueprint.properties = properties

            # Create the host definitions
            host_definition_field = self.fields['host_definitions']
            # Add in the blueprint to all the host defs
            for host_definition in host_definitions:
                host_definition['blueprint'] = blueprint
            host_definition_field.create(host_definitions)

            # Create the formula versions
            formula_version_field = self.fields['formula_versions']
            # Add in the blueprint to all the formula versions
            for formula_version in formula_versions:
                formula_version['content_object'] = blueprint
            formula_version_field.create(formula_versions)

            # Create the labels
            label_field = self.fields['labels']
            # Add in the blueprint to all the labels
            for label in labels:
                label['content_object'] = blueprint
            label_field.create(labels)

        # Add the other fields back in for deserialization
        validated_data['properties'] = properties
        validated_data['host_definitions'] = host_definitions
        validated_data['formula_versions'] = formula_versions
        validated_data['labels'] = labels

        return blueprint

class BlueprintExportSerializer(FullBlueprintSerializer):
    class Meta:
        model = models.Blueprint
        fields = (
            'title',
            'description',
            'create_users',
            'properties',
            'labels',
            'host_definitions',
            'formula_versions',
        )

class BlueprintLabelSerializer(StackdioLabelSerializer):

    class Meta(StackdioLabelSerializer.Meta):
        app_label = 'blueprints'
        model_name = 'blueprint-label'
<|endoftext|>


<commit_before>#Define the critical functions used in the program
# These function should almost always return a value

def mul(x,y):
	return 2 * (x + y)

def output(name, x, y, z):
	out = """
HEY GUYS
My name is {}.
I am your nurse.
Your baby girl might have {} hairs on the top of her head.
Oops. We just detected she might have {} hairs on her face as well.
We are afraid that if she has more than {} hairs on her face and more than {} hairs on her head...
She could be diagnosed with hypertrichosis.
However, if are able to wax and shave {} amount of hair on her face.
We might get all but {} hairs on her face.
I'm sorry but that is the best we can do.
Hope your daughter doesn't grow up and get bullied.
""".format(name, x, y, z)
	return out

def main():
	name= raw_input("What's your name?: ")
	x = raw_input("Type your brother's age: ")
	y = raw_input("Type your age: ")
	z = add(int(x), int(y))
	out = output(name, x, y, z)
	print out

main()

<commit_msg>I edited my file.<commit_after>#Define the critical functions used in the program
# These function should almost always return a value

def bothdaughtershairscombined(x,y):
	return 2 * (x + y)

def output(name, x, y, z):
	out = """
HEY GUYS
My name is {}.
I am your nurse.
Your baby girl might have {} hairs on her face.
We are afraid that if she has more than {} hairs on her face...
Then she could be diagnosed with hypertrichosis.
So if we add both of your daughter's amount of hair on the head and face, then we have {} amount of hair for both daughters together.
This means really need to try and shave your daughter's face with hypertrichosis to have an even number of hairs for both children. We should end up with an easy 100.
""".format(name, x, y, z)
	return out

def main():
	name= raw_input("What's your name?: ")
	x = raw_input("Type your daughter's amount of hair on her face with hypertrichosis: ")
	y = raw_input("Type the amount of hair she should have on her face: ")
	z = bothdaughtershairscombined(int(x), int(y))
	out = output(name, x, y, z)
	print out

main()
<|endoftext|>


<commit_before>###########################################################################
#
#  Copyright 2020 Google LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
###########################################################################

def rgb_to_hsv():
  return """
  CREATE FUNCTION IF NOT EXISTS rgb_to_hsv(r FLOAT64, g FLOAT64, b FLOAT64)
    RETURNS STRUCT<h int64, s int64, v int64>
    LANGUAGE js AS r\'\'\'
      var h = 0;
      var s = 0;
      var v = 0;
      r /= 255.0;
      g /= 255.0;
      b /= 255.0;
      var mx = Math.max(r, g, b);
      var mn = Math.min(r, g, b);
      var df = mx-mn;

      if (mx == mn) { h = 0; }
      else if (mx == r) { h = (60 * ((g-b)/df) + 360) % 360; }
      else if (mx == g) { h = (60 * ((b-r)/df) + 120) % 360; }
      else if (mx == b) { h = (60 * ((r-g)/df) + 240) % 360; }

      if (mx > 0) { s = (df/mx)*100; }

      v = mx*100;

      return {'h':h, 's':s, 'v':v};
    \'\'\'
  """

def pearson_significance_test():
  return """
  CREATE FUNCTION IF NOT EXISTS pearson_significance_test(correlation FLOAT64, population INT64, percent FLOAT64, threshold FLOAT64)
    RETURNS INT64
    LANGUAGE js AS
    \'\'\'
      /* correlation = value from CORR function. */
      /* population = number of mesurements. */
      /* percent = 80, 90, 98, 99, 99.5, 99.95 certainty ( aka sigma ). */
      /* threshold = usually 0.5, the minimum correlation value to be considered significant. */
      /* returns = -1 for negative significant correlation, 0 for no significant correlation ( null hypothesis), and 1 for positive siginificant correlation. */
      /* Example: SELECT pearson_significance_test(Impression_Correlation, Data_Point_Count, 95, 0.5) AS Female_Impression_Significant FROM ... */
      if (Math.abs(correlation) < threshold) { return 0; }
      var dt_lookup = [
        [1, .9511, .9877, .9969, .9995, .9999, .9999], /* 0 */
        [2, .8000, .9000, .9500, .9800, .9900, .9990], /* 1 */
        [3, .6870, .8054, .8783, .9343, .9587, .9911], /* 2 */
        [4, .6084, .7293, .8114, .8822, .9172, .9741], /* 3 */
        [5, .5509, .6694, .7545, .8329, .8745, .9509], /* 4 */
        [6, .5067, .6215, .7067, .7887, .8343, .9249], /* 5 */
        [7, .4716, .5822, .6664, .7498, .7977, .8983], /* 6 */
        [8, .4428, .5494, .6319, .7155, .7646, .8721], /* 7 */
        [9, .4187, .5214, .6021, .6851, .7348, .8470], /* 8 */
        [10, .3981, .4973, .5760, .6581, .7079, .8233], /* 9 */
        [11, .3802, .4762, .5529, .6339, .6835, .8010], /* 10 */
        [12, .3646, .4575, .5324, .6120, .6614, .7800], /* 11 */
        [13, .3507, .4409, .5140, .5923, .6411, .7604], /* 12 */
        [14, .3383, .4259, .4973, .5742, .6226, .7419], /* 13 */
        [15, .3271, .4124, .4821, .5577, .6055, .7247], /* 14 */
        [16, .3170, .4000, .4683, .5425, .5897, .7084], /* 15 */
        [17, .3077, .3887, .4555, .5285, .5751, .6932], /* 16 */
        [18, .2992, .3783, .4438, .5155, .5614, .6788], /* 17 */
        [19, .2914, .3687, .4329, .5034, .5487, .6652], /* 18 */
        [20, .2841, .3598, .4227, .4921, .5368, .6524], /* 29 */
        [21, .2774, .3515, .4132, .4815, .5256, .6402], /* 20 */
        [22, .2711, .3438, .4044, .4716, .5151, .6287], /* 21 */
        [23, .2653, .3365, .3961, .4622, .5052, .6178], /* 22 */
        [24, .2598, .3297, .3882, .4534, .4958, .6074], /* 23 */
        [25, .2546, .3233, .3809, .4451, .4869, .5974], /* 24 */
        [30, .2327, .2960, .3494, .4093, .4487, .5541], /* 25 */
        [35, .2156, .2746, .3246, .3810, .4182, .5189], /* 26 */
        [40, .2018, .2573, .3044, .3578, .3932, .4896], /* 27 */
        [50, .1806, .2306, .2732, .3218, .3542, .4432], /* 28 */
        [60, .1650, .2108, .2500, .2948, .3248, .4079], /* 29 */
        [70, .1528, .1954, .2319, .2737, .3017, .3798], /* 30 */
        [80, .1430, .1829, .2172, .2565, .2830, .3568], /* 31 */
        [90, .1348, .1726, .2050, .2422, .2673, .3375], /* 32 */
        [100, .1279, .1638, .1946, .2301, .2540, .3211], /* 33 */
        [150, .1045, .1339, .1593, .1886, .2084, .2643], /* 34 */
        [300, .0740, .0948, .1129, .1338, .1480, .1884], /* 35 */
        [500, .0573, .0735, .0875, .1038, .1149, .1464], /* 36 */
        [1000, .0405, .0520, .0619, .0735, .0813, .1038] /* 37 */
      ]
      populatuion = population - 2;
      var dt_low = 0;
      var dt_high = 0;
      var dt_sigma = 0;
      if (percent == 80) { dt_sigma = 1; }
      else if (percent == 90) { dt_sigma = 2; }
      else if (percent == 95) { dt_sigma = 3; }
      else if (percent == 98) { dt_sigma = 4; }
      else if (percent == 99.5) { dt_sigma = 5; }
      else if (percent == 99.95) { dt_sigma = 6; }
      else { dt_sigma = 5; }
      if (population >= 1000) { dt_low = 37; dt_high = 37; }
      else if (population >= 500) { dt_low = 36; dt_high = 37; }
      else if (population >= 300) { dt_low = 35; dt_high = 36; }
      else if (population >= 150) { dt_low = 34; dt_high = 35; }
      else if (population >= 100) { dt_low = 33; dt_high = 34; }
      else if (population >= 90) { dt_low = 32; dt_high = 33; }
      else if (population >= 80) { dt_low = 31; dt_high = 32; }
      else if (population >= 70) { dt_low = 30; dt_high = 31; }
      else if (population >= 60) { dt_low = 29; dt_high = 30; }
      else if (population >= 50) { dt_low = 28; dt_high = 29; }
      else if (population >= 40) { dt_low = 27; dt_high = 28; }
      else if (population >= 35) { dt_low = 26; dt_high = 27; }
      else if (population >= 30) { dt_low = 25; dt_high = 26; }
      else if (population > 25) { dt_low = 24; dt_high = 25; }
      else if (population > 0) { dt_low = population; dt_high = population; }
      else { return 0; }
      if (dt_low == dt_high) {
        if (Math.abs(correlation) > dt_lookup[dt_low][dt_sigma] ) { return Math.sign(correlation); }
        else { return 0; }
      } else {
        if (Math.abs(correlation) > dt_lookup[dt_low][dt_sigma] + ((dt_lookup[dt_high][dt_sigma] - dt_lookup[dt_low][dt_sigma]) * ((population - dt_lookup[dt_low][0]) / (dt_lookup[dt_high][0] - dt_lookup[dt_low][0])))) { return Math.sign(correlation); }
        else { return 0; }
      }
    \'\'\'
  """
<commit_msg>Make all non standard values default to 80%.<commit_after>###########################################################################
#
#  Copyright 2020 Google LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
###########################################################################

def rgb_to_hsv():
  return """
  CREATE FUNCTION IF NOT EXISTS rgb_to_hsv(r FLOAT64, g FLOAT64, b FLOAT64)
    RETURNS STRUCT<h int64, s int64, v int64>
    LANGUAGE js AS r\'\'\'
      var h = 0;
      var s = 0;
      var v = 0;
      r /= 255.0;
      g /= 255.0;
      b /= 255.0;
      var mx = Math.max(r, g, b);
      var mn = Math.min(r, g, b);
      var df = mx-mn;

      if (mx == mn) { h = 0; }
      else if (mx == r) { h = (60 * ((g-b)/df) + 360) % 360; }
      else if (mx == g) { h = (60 * ((b-r)/df) + 120) % 360; }
      else if (mx == b) { h = (60 * ((r-g)/df) + 240) % 360; }

      if (mx > 0) { s = (df/mx)*100; }

      v = mx*100;

      return {'h':h, 's':s, 'v':v};
    \'\'\'
  """

def pearson_significance_test():
  return """
  CREATE FUNCTION IF NOT EXISTS pearson_significance_test(correlation FLOAT64, population INT64, percent FLOAT64, threshold FLOAT64)
    RETURNS INT64
    LANGUAGE js AS
    \'\'\'
      /* correlation = value from CORR function. */
      /* population = number of mesurements. */
      /* percent = 80, 90, 98, 99, 99.5, 99.95 certainty ( aka sigma ). */
      /* threshold = usually 0.5, the minimum correlation value to be considered significant. */
      /* returns = -1 for negative significant correlation, 0 for no significant correlation ( null hypothesis), and 1 for positive siginificant correlation. */
      /* Example: SELECT pearson_significance_test(Impression_Correlation, Data_Point_Count, 95, 0.5) AS Female_Impression_Significant FROM ... */
      if (Math.abs(correlation) < threshold) { return 0; }
      var dt_lookup = [
        [1, .9511, .9877, .9969, .9995, .9999, .9999], /* 0 */
        [2, .8000, .9000, .9500, .9800, .9900, .9990], /* 1 */
        [3, .6870, .8054, .8783, .9343, .9587, .9911], /* 2 */
        [4, .6084, .7293, .8114, .8822, .9172, .9741], /* 3 */
        [5, .5509, .6694, .7545, .8329, .8745, .9509], /* 4 */
        [6, .5067, .6215, .7067, .7887, .8343, .9249], /* 5 */
        [7, .4716, .5822, .6664, .7498, .7977, .8983], /* 6 */
        [8, .4428, .5494, .6319, .7155, .7646, .8721], /* 7 */
        [9, .4187, .5214, .6021, .6851, .7348, .8470], /* 8 */
        [10, .3981, .4973, .5760, .6581, .7079, .8233], /* 9 */
        [11, .3802, .4762, .5529, .6339, .6835, .8010], /* 10 */
        [12, .3646, .4575, .5324, .6120, .6614, .7800], /* 11 */
        [13, .3507, .4409, .5140, .5923, .6411, .7604], /* 12 */
        [14, .3383, .4259, .4973, .5742, .6226, .7419], /* 13 */
        [15, .3271, .4124, .4821, .5577, .6055, .7247], /* 14 */
        [16, .3170, .4000, .4683, .5425, .5897, .7084], /* 15 */
        [17, .3077, .3887, .4555, .5285, .5751, .6932], /* 16 */
        [18, .2992, .3783, .4438, .5155, .5614, .6788], /* 17 */
        [19, .2914, .3687, .4329, .5034, .5487, .6652], /* 18 */
        [20, .2841, .3598, .4227, .4921, .5368, .6524], /* 29 */
        [21, .2774, .3515, .4132, .4815, .5256, .6402], /* 20 */
        [22, .2711, .3438, .4044, .4716, .5151, .6287], /* 21 */
        [23, .2653, .3365, .3961, .4622, .5052, .6178], /* 22 */
        [24, .2598, .3297, .3882, .4534, .4958, .6074], /* 23 */
        [25, .2546, .3233, .3809, .4451, .4869, .5974], /* 24 */
        [30, .2327, .2960, .3494, .4093, .4487, .5541], /* 25 */
        [35, .2156, .2746, .3246, .3810, .4182, .5189], /* 26 */
        [40, .2018, .2573, .3044, .3578, .3932, .4896], /* 27 */
        [50, .1806, .2306, .2732, .3218, .3542, .4432], /* 28 */
        [60, .1650, .2108, .2500, .2948, .3248, .4079], /* 29 */
        [70, .1528, .1954, .2319, .2737, .3017, .3798], /* 30 */
        [80, .1430, .1829, .2172, .2565, .2830, .3568], /* 31 */
        [90, .1348, .1726, .2050, .2422, .2673, .3375], /* 32 */
        [100, .1279, .1638, .1946, .2301, .2540, .3211], /* 33 */
        [150, .1045, .1339, .1593, .1886, .2084, .2643], /* 34 */
        [300, .0740, .0948, .1129, .1338, .1480, .1884], /* 35 */
        [500, .0573, .0735, .0875, .1038, .1149, .1464], /* 36 */
        [1000, .0405, .0520, .0619, .0735, .0813, .1038] /* 37 */
      ]
      populatuion = population - 2;
      var dt_low = 0;
      var dt_high = 0;
      var dt_sigma = 0;
      if (percent <= 80) { dt_sigma = 1; }
      else if (percent <= 90) { dt_sigma = 2; }
      else if (percent <= 95) { dt_sigma = 3; }
      else if (percent <= 98) { dt_sigma = 4; }
      else if (percent <= 99.5) { dt_sigma = 5; }
      else if (percent <= 99.95) { dt_sigma = 6; }
      else { dt_sigma = 6; }
      if (population >= 1000) { dt_low = 37; dt_high = 37; }
      else if (population >= 500) { dt_low = 36; dt_high = 37; }
      else if (population >= 300) { dt_low = 35; dt_high = 36; }
      else if (population >= 150) { dt_low = 34; dt_high = 35; }
      else if (population >= 100) { dt_low = 33; dt_high = 34; }
      else if (population >= 90) { dt_low = 32; dt_high = 33; }
      else if (population >= 80) { dt_low = 31; dt_high = 32; }
      else if (population >= 70) { dt_low = 30; dt_high = 31; }
      else if (population >= 60) { dt_low = 29; dt_high = 30; }
      else if (population >= 50) { dt_low = 28; dt_high = 29; }
      else if (population >= 40) { dt_low = 27; dt_high = 28; }
      else if (population >= 35) { dt_low = 26; dt_high = 27; }
      else if (population >= 30) { dt_low = 25; dt_high = 26; }
      else if (population > 25) { dt_low = 24; dt_high = 25; }
      else if (population > 0) { dt_low = population; dt_high = population; }
      else { return 0; }
      if (dt_low == dt_high) {
        if (Math.abs(correlation) > dt_lookup[dt_low][dt_sigma] ) { return Math.sign(correlation); }
        else { return 0; }
      } else {
        if (Math.abs(correlation) > dt_lookup[dt_low][dt_sigma] + ((dt_lookup[dt_high][dt_sigma] - dt_lookup[dt_low][dt_sigma]) * ((population - dt_lookup[dt_low][0]) / (dt_lookup[dt_high][0] - dt_lookup[dt_low][0])))) { return Math.sign(correlation); }
        else { return 0; }
      }
    \'\'\'
  """
<|endoftext|>


<commit_before># *****************************************************************************
# Copyright (c) 2020, Intel Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#
#     Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# *****************************************************************************

import os
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pandas as pd

class ParquetGenerator:
    GEN_KDE_PQ_CALLED = False
    GEN_PQ_TEST_CALLED = False

    @classmethod
    def gen_kde_pq(cls, file_name='kde.parquet', N=101):
        if not cls.GEN_KDE_PQ_CALLED:
            df = pd.DataFrame({'points': np.random.random(N)})
            table = pa.Table.from_pandas(df)
            row_group_size = 128
            pq.write_table(table, file_name, row_group_size)
            cls.GEN_KDE_PQ_CALLED = True

    @classmethod
    def gen_pq_test(cls):
        if not cls.GEN_PQ_TEST_CALLED:
            df = pd.DataFrame(
                {
                    'one': [-1, np.nan, 2.5, 3., 4., 6., 10.0],
                    'two': ['foo', 'bar', 'baz', 'foo', 'bar', 'baz', 'foo'],
                    'three': [True, False, True, True, True, False, False],
                    # float without NA
                    'four': [-1, 5.1, 2.5, 3., 4., 6., 11.0],
                    # str with NA
                    'five': ['foo', 'bar', 'baz', None, 'bar', 'baz', 'foo'],
                }
            )
            table = pa.Table.from_pandas(df)
            pq.write_table(table, 'example.parquet')
            pq.write_table(table, 'example2.parquet', row_group_size=2)
            cls.GEN_PQ_TEST_CALLED = True

def generate_spark_data():
    # test datetime64, spark dates
    dt1 = pd.DatetimeIndex(['2017-03-03 03:23',
                            '1990-10-23', '1993-07-02 10:33:01'])
    df = pd.DataFrame({'DT64': dt1, 'DATE': dt1.copy()})
    df.to_parquet('pandas_dt.pq')

    import os
    import shutil
    import tarfile

    if os.path.exists('sdf_dt.pq'):
        shutil.rmtree('sdf_dt.pq')

    sdf_dt_archive = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sdf_dt.pq.bz2')
    tar = tarfile.open(sdf_dt_archive, "r:bz2")
    tar.extractall('.')
    tar.close()

def generate_csv_data():
    # CSV reader test
    data = ("0,2.3,4.6,A\n"
            "1,2.3,4.6,B\n"
            "2,2.3,4.6,\n"
            "4,2.3,4.6,D\n")

    with open("csv_data1.csv", "w") as f:
        f.write(data)

    with open("csv_data2.csv", "w") as f:
        # the same types as csv_data1.csv
        f.write((
            "4,2.1,4.1,D\n"
            "3,2.2,4.2,\n"
            "2,2.3,4.3,B\n"
            "1,2.4,4.4,A\n"
        ))

    with open("csv_data_infer1.csv", "w") as f:
        f.write('A,B,C,D\n' + data)

    with open("csv_data_infer_no_column_name.csv", "w") as f:
        f.write(',,C,D\n' + data)

    with open("csv_data_infer_sep.csv", "w") as f:
        f.write(('A,B,C,D\n' + data).replace(',', ';'))

    data = ("0,2.3,2015-01-03,47736\n"
            "1,2.3,1966-11-13,47736\n"
            "2,2.3,1998-05-21,47736\n"
            "4,2.3,2018-07-11,47736\n")

    with open("csv_data_date1.csv", "w") as f:
        f.write(data)

    # generated data for parallel merge_asof testing
    df1 = pd.DataFrame({'time': pd.DatetimeIndex(
        ['2017-01-03', '2017-01-06', '2017-02-15', '2017-02-21']),
        'B': [4, 5, 9, 6]})
    df2 = pd.DataFrame({'time': pd.DatetimeIndex(
        ['2017-01-01', '2017-01-14', '2017-01-16', '2017-02-23', '2017-02-23',
         '2017-02-25']), 'A': [2, 3, 7, 8, 9, 10]})
    df1.to_parquet("asof1.pq")
    df2.to_parquet("asof2.pq")

def generate_other_data():
    df = pd.DataFrame({'A': ['bc'] + ["a"] * 3 + ["bc"] * 3 + ['a'], 'B': [-8, 1, 2, 3, 1, 5, 6, 7]})
    df.to_parquet("groupby3.pq")

    df = pd.DataFrame({"A": ["foo", "foo", "foo", "foo", "foo",
                             "bar", "bar", "bar", "bar"],
                       "B": ["one", "one", "one", "two", "two",
                             "one", "one", "two", "two"],
                       "C": ["small", "large", "large", "small",
                             "small", "large", "small", "small",
                             "large"],
                       "D": [1, 2, 2, 6, 3, 4, 5, 6, 9]})
    df.to_parquet("pivot2.pq")

    # generated data for parallel merge_asof testing
    df1 = pd.DataFrame({'time': pd.DatetimeIndex(
        ['2017-01-03', '2017-01-06', '2017-02-15', '2017-02-21']),
        'B': [4, 5, 9, 6]})
    df2 = pd.DataFrame({'time': pd.DatetimeIndex(
        ['2017-01-01', '2017-01-14', '2017-01-16', '2017-02-23', '2017-02-23',
         '2017-02-25']), 'A': [2, 3, 7, 8, 9, 10]})
    df1.to_parquet("asof1.pq")
    df2.to_parquet("asof2.pq")

if __name__ == "__main__":
    print(f'generation phase in {os.getcwd()}')
    ParquetGenerator.gen_kde_pq()
    ParquetGenerator.gen_pq_test()
    generate_spark_data()
    generate_csv_data()
    generate_other_data()
<commit_msg>SAT-2941 - SDC: Function tests failed for all platforms<commit_after># *****************************************************************************
# Copyright (c) 2020, Intel Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#
#     Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# *****************************************************************************

import os
import numpy as np
import pyarrow as pa
# import pyarrow.parquet as pq
import pandas as pd

class ParquetGenerator:
    GEN_KDE_PQ_CALLED = False
    GEN_PQ_TEST_CALLED = False

    @classmethod
    def gen_kde_pq(cls, file_name='kde.parquet', N=101):
        if not cls.GEN_KDE_PQ_CALLED:
            df = pd.DataFrame({'points': np.random.random(N)})
            table = pa.Table.from_pandas(df)
            row_group_size = 128
            pq.write_table(table, file_name, row_group_size)
            cls.GEN_KDE_PQ_CALLED = True

    @classmethod
    def gen_pq_test(cls):
        if not cls.GEN_PQ_TEST_CALLED:
            df = pd.DataFrame(
                {
                    'one': [-1, np.nan, 2.5, 3., 4., 6., 10.0],
                    'two': ['foo', 'bar', 'baz', 'foo', 'bar', 'baz', 'foo'],
                    'three': [True, False, True, True, True, False, False],
                    # float without NA
                    'four': [-1, 5.1, 2.5, 3., 4., 6., 11.0],
                    # str with NA
                    'five': ['foo', 'bar', 'baz', None, 'bar', 'baz', 'foo'],
                }
            )
            table = pa.Table.from_pandas(df)
            pq.write_table(table, 'example.parquet')
            pq.write_table(table, 'example2.parquet', row_group_size=2)
            cls.GEN_PQ_TEST_CALLED = True

def generate_spark_data():
    # test datetime64, spark dates
    dt1 = pd.DatetimeIndex(['2017-03-03 03:23',
                            '1990-10-23', '1993-07-02 10:33:01'])
    df = pd.DataFrame({'DT64': dt1, 'DATE': dt1.copy()})
    df.to_parquet('pandas_dt.pq')

    import os
    import shutil
    import tarfile

    if os.path.exists('sdf_dt.pq'):
        shutil.rmtree('sdf_dt.pq')

    sdf_dt_archive = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sdf_dt.pq.bz2')
    tar = tarfile.open(sdf_dt_archive, "r:bz2")
    tar.extractall('.')
    tar.close()

def generate_csv_data():
    # CSV reader test
    data = ("0,2.3,4.6,A\n"
            "1,2.3,4.6,B\n"
            "2,2.3,4.6,\n"
            "4,2.3,4.6,D\n")

    with open("csv_data1.csv", "w") as f:
        f.write(data)

    with open("csv_data2.csv", "w") as f:
        # the same types as csv_data1.csv
        f.write((
            "4,2.1,4.1,D\n"
            "3,2.2,4.2,\n"
            "2,2.3,4.3,B\n"
            "1,2.4,4.4,A\n"
        ))

    with open("csv_data_infer1.csv", "w") as f:
        f.write('A,B,C,D\n' + data)

    with open("csv_data_infer_no_column_name.csv", "w") as f:
        f.write(',,C,D\n' + data)

    with open("csv_data_infer_sep.csv", "w") as f:
        f.write(('A,B,C,D\n' + data).replace(',', ';'))

    data = ("0,2.3,2015-01-03,47736\n"
            "1,2.3,1966-11-13,47736\n"
            "2,2.3,1998-05-21,47736\n"
            "4,2.3,2018-07-11,47736\n")

    with open("csv_data_date1.csv", "w") as f:
        f.write(data)

    # generated data for parallel merge_asof testing
    df1 = pd.DataFrame({'time': pd.DatetimeIndex(
        ['2017-01-03', '2017-01-06', '2017-02-15', '2017-02-21']),
        'B': [4, 5, 9, 6]})
    df2 = pd.DataFrame({'time': pd.DatetimeIndex(
        ['2017-01-01', '2017-01-14', '2017-01-16', '2017-02-23', '2017-02-23',
         '2017-02-25']), 'A': [2, 3, 7, 8, 9, 10]})
    df1.to_parquet("asof1.pq")
    df2.to_parquet("asof2.pq")

def generate_other_data():
    df = pd.DataFrame({'A': ['bc'] + ["a"] * 3 + ["bc"] * 3 + ['a'], 'B': [-8, 1, 2, 3, 1, 5, 6, 7]})
    df.to_parquet("groupby3.pq")

    df = pd.DataFrame({"A": ["foo", "foo", "foo", "foo", "foo",
                             "bar", "bar", "bar", "bar"],
                       "B": ["one", "one", "one", "two", "two",
                             "one", "one", "two", "two"],
                       "C": ["small", "large", "large", "small",
                             "small", "large", "small", "small",
                             "large"],
                       "D": [1, 2, 2, 6, 3, 4, 5, 6, 9]})
    df.to_parquet("pivot2.pq")

    # generated data for parallel merge_asof testing
    df1 = pd.DataFrame({'time': pd.DatetimeIndex(
        ['2017-01-03', '2017-01-06', '2017-02-15', '2017-02-21']),
        'B': [4, 5, 9, 6]})
    df2 = pd.DataFrame({'time': pd.DatetimeIndex(
        ['2017-01-01', '2017-01-14', '2017-01-16', '2017-02-23', '2017-02-23',
         '2017-02-25']), 'A': [2, 3, 7, 8, 9, 10]})
    df1.to_parquet("asof1.pq")
    df2.to_parquet("asof2.pq")

if __name__ == "__main__":
    print(f'generation phase in {os.getcwd()}')
    # ParquetGenerator.gen_kde_pq()
    # ParquetGenerator.gen_pq_test()
    # generate_spark_data()
    generate_csv_data()
    # generate_other_data()
<|endoftext|>