<commit_before>#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test RPCs related to blockchainstate.

Test the following RPCs:
    - gettxoutsetinfo
    - verifychain

Tests correspond to code in rpc/blockchain.cpp.
"""

from decimal import Decimal

from test_framework.test_framework import BitcoinTestFramework
from test_framework.authproxy import JSONRPCException
from test_framework.util import (
    assert_equal,
    assert_raises,
    assert_is_hex_string,
    assert_is_hash_string,
    start_nodes,
    connect_nodes_bi,
)

class BlockchainTest(BitcoinTestFramework):

    def __init__(self):
        super().__init__()
        self.setup_clean_chain = False
        self.num_nodes = 1

    def setup_network(self, split=False):
        self.nodes = start_nodes(self.num_nodes, self.options.tmpdir)
        self.is_network_split = False
        self.sync_all()

    def run_test(self):
        self._test_gettxoutsetinfo()
        self._test_getblockheader()
        self.nodes[0].verifychain(4, 0)

    def _test_gettxoutsetinfo(self):
        node = self.nodes[0]
        res = node.gettxoutsetinfo()

        assert_equal(res['total_amount'], Decimal('98214.28571450'))
        assert_equal(res['transactions'], 200)
        assert_equal(res['height'], 200)
        assert_equal(res['txouts'], 200)
        size = res['disk_size']
        assert size > 6400
        assert size < 64000
        assert_equal(len(res['bestblock']), 64)
        assert_equal(len(res['hash_serialized_2']), 64)

        print("Test that gettxoutsetinfo() works for blockchain with just the genesis block")
        b1hash = node.getblockhash(1)
        node.invalidateblock(b1hash)

        res2 = node.gettxoutsetinfo()
        assert_equal(res2['transactions'], 0)
        assert_equal(res2['total_amount'], Decimal('0'))
        assert_equal(res2['height'], 0)
        assert_equal(res2['txouts'], 0)
        assert_equal(res2['bestblock'], node.getblockhash(0))
        assert_equal(len(res2['hash_serialized_2']), 64)

        print("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block")
        node.reconsiderblock(b1hash)

        res3 = node.gettxoutsetinfo()
        assert_equal(res['total_amount'], res3['total_amount'])
        assert_equal(res['transactions'], res3['transactions'])
        assert_equal(res['height'], res3['height'])
        assert_equal(res['txouts'], res3['txouts'])
        assert_equal(res['bestblock'], res3['bestblock'])
        assert_equal(res['hash_serialized_2'], res3['hash_serialized_2'])

    def _test_getblockheader(self):
        node = self.nodes[0]

        assert_raises(
            JSONRPCException, lambda: node.getblockheader('nonsense'))

        besthash = node.getbestblockhash()
        secondbesthash = node.getblockhash(199)
        header = node.getblockheader(besthash)

        assert_equal(header['hash'], besthash)
        assert_equal(header['height'], 200)
        assert_equal(header['confirmations'], 1)
        assert_equal(header['previousblockhash'], secondbesthash)
        assert_is_hex_string(header['chainwork'])
        assert_is_hash_string(header['hash'])
        assert_is_hash_string(header['previousblockhash'])
        assert_is_hash_string(header['merkleroot'])
        assert_is_hash_string(header['bits'], length=None)
        assert isinstance(header['time'], int)
        assert isinstance(header['mediantime'], int)
        assert isinstance(header['nonce'], int)
        assert isinstance(header['version'], int)
        assert isinstance(int(header['versionHex'], 16), int)
        assert isinstance(header['difficulty'], Decimal)

if __name__ == '__main__':
    BlockchainTest().main()
<commit_msg>Revert "Fix use of missing self.log in blockchain.py"<commit_after>#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test RPCs related to blockchainstate.

Test the following RPCs:
    - gettxoutsetinfo
    - verifychain

Tests correspond to code in rpc/blockchain.cpp.
"""

from decimal import Decimal

from test_framework.test_framework import BitcoinTestFramework
from test_framework.authproxy import JSONRPCException
from test_framework.util import (
    assert_equal,
    assert_raises,
    assert_is_hex_string,
    assert_is_hash_string,
    start_nodes,
    connect_nodes_bi,
)

class BlockchainTest(BitcoinTestFramework):

    def __init__(self):
        super().__init__()
        self.setup_clean_chain = False
        self.num_nodes = 1

    def setup_network(self, split=False):
        self.nodes = start_nodes(self.num_nodes, self.options.tmpdir)
        self.is_network_split = False
        self.sync_all()

    def run_test(self):
        self._test_gettxoutsetinfo()
        self._test_getblockheader()
        self.nodes[0].verifychain(4, 0)

    def _test_gettxoutsetinfo(self):
        node = self.nodes[0]
        res = node.gettxoutsetinfo()

        assert_equal(res['total_amount'], Decimal('98214.28571450'))
        assert_equal(res['transactions'], 200)
        assert_equal(res['height'], 200)
        assert_equal(res['txouts'], 200)
        size = res['disk_size']
        assert size > 6400
        assert size < 64000
        assert_equal(len(res['bestblock']), 64)
        assert_equal(len(res['hash_serialized_2']), 64)

        self.log.info("Test that gettxoutsetinfo() works for blockchain with just the genesis block")
        b1hash = node.getblockhash(1)
        node.invalidateblock(b1hash)

        res2 = node.gettxoutsetinfo()
        assert_equal(res2['transactions'], 0)
        assert_equal(res2['total_amount'], Decimal('0'))
        assert_equal(res2['height'], 0)
        assert_equal(res2['txouts'], 0)
        assert_equal(res2['bestblock'], node.getblockhash(0))
        assert_equal(len(res2['hash_serialized_2']), 64)

        self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block")
        node.reconsiderblock(b1hash)

        res3 = node.gettxoutsetinfo()
        assert_equal(res['total_amount'], res3['total_amount'])
        assert_equal(res['transactions'], res3['transactions'])
        assert_equal(res['height'], res3['height'])
        assert_equal(res['txouts'], res3['txouts'])
        assert_equal(res['bestblock'], res3['bestblock'])
        assert_equal(res['hash_serialized_2'], res3['hash_serialized_2'])

    def _test_getblockheader(self):
        node = self.nodes[0]

        assert_raises(
            JSONRPCException, lambda: node.getblockheader('nonsense'))

        besthash = node.getbestblockhash()
        secondbesthash = node.getblockhash(199)
        header = node.getblockheader(besthash)

        assert_equal(header['hash'], besthash)
        assert_equal(header['height'], 200)
        assert_equal(header['confirmations'], 1)
        assert_equal(header['previousblockhash'], secondbesthash)
        assert_is_hex_string(header['chainwork'])
        assert_is_hash_string(header['hash'])
        assert_is_hash_string(header['previousblockhash'])
        assert_is_hash_string(header['merkleroot'])
        assert_is_hash_string(header['bits'], length=None)
        assert isinstance(header['time'], int)
        assert isinstance(header['mediantime'], int)
        assert isinstance(header['nonce'], int)
        assert isinstance(header['version'], int)
        assert isinstance(int(header['versionHex'], 16), int)
        assert isinstance(header['difficulty'], Decimal)

if __name__ == '__main__':
    BlockchainTest().main()
<|endoftext|>


<commit_before><commit_msg>Fix test cases for exports<commit_after><|endoftext|>


<commit_before><commit_msg>Add argument --project-name to diff command<commit_after><|endoftext|>


<commit_before>"""
    drfjsonapi.serializers
    ~~~~~~~~~~~~~~~~~~~~~~~

    DRF serializers to assist with a JSON API spec compliant API.

    Specifically, the JsonApiRenderer requires the use of the
    JsonApiSerializer
"""

from django.core.exceptions import ImproperlyConfigured
from rest_framework import exceptions
from rest_framework import serializers
from rest_framework.relations import ManyRelatedField
from .exceptions import (
    FieldError,
    ManyExceptions,
    RelationshipError,
    ResourceError,
    RtypeConflict,
)
from .relations import ResourceRelatedField
from .utils import _get_resource_url

# pylint: disable=abstract-method
class JsonApiSerializer(serializers.Serializer):
    """ JSON API Serializer """

    @property
    def related_fields(self):
        """ Return `self.fields` but limited to relationship fields """

        fields = {}
        for name, field in self.fields.items():
            if field.write_only:
                continue
            elif isinstance(field, ManyRelatedField):
                fields[name] = field.child_relation
            elif isinstance(field, ResourceRelatedField):
                fields[name] = field
        return fields

    def get_filterable_fields(self):
        """ Return a dict of fields allowed to be filtered on

        By default the `JsonApiMeta.filterable_fields` property
        is used to source the items.

        The key of each item is the string name of the field &
        the value is the `FilterField` object instance.
        """

        try:
            return self.JsonApiMeta.filterable_fields
        except AttributeError:
            return {}

    def get_default_include_fields(self):
        """ Return the default related fields to include

        This is only used if the requestor did not explicitly
        request includes per the JSON API spec.
        """

        try:
            return {
                k: v for k, v in self.get_includable_fields.items()
                if k in self.JsonApiMeta.default_include_fields
            }
        except (AttributeError, TypeError):
            return {}

    def get_includable_fields(self):
        """ Return includable related fields """

        try:
            return self.JsonApiMeta.includable_fields
        except AttributeError:
            return {}

    def get_links(self, instance):
        """ Return the "Links" object for an individual resource

        This should return a dict that is compliant with the
        document links section of the JSON API spec. Specifically,
        the links section of an individual "Resource Object".

        :spec:
            jsonapi.org/format/#document-links
        """

        rtype = self.get_rtype()
        resource_url = _get_resource_url(rtype, instance.id, self.context)
        return {'self': resource_url}

    def get_meta(self):
        """ Return the "Meta" object for an individual resource

        This should return a dict that is compliant with the
        document meta section of the JSON API spec. Specifically,
        the links section of an individual "Resource Object".

        :spec:
            jsonapi.org/format/#document-meta
        """

        return {}

    def get_relationships(self, data):
        """ Return the "Relationships Object" for a resource

        This should return a dict that is compliant with the
        "Relationships Object" section of the JSON API spec.
        Specifically, the contents of the top level `relationships`
        member of an individual resource boject.

        Relationships always get top-level links & meta objects
        but if they have been included or require "Linkage" then
        the "Resource Linkage" is added to the object as well.

        Often to-one relationships will have linkage=True while
        to-many's won't since there could be alot of them.

        NOTE: if the field is included then we don't worry
              about it here because the IncludesFilter will
              process all those defaults & populate the context's
              `includes` keys.

        :spec:
            jsonapi.org/format/#document-resource-object-relationships
        """

        relationships = {}
        for key, field in self.related_fields.items():
            relationships[key] = {
                'data': data.pop(key),
                'links': field.get_links(data['id']),
                'meta': field.get_meta(),
            }

            includes = self.context.get('includes', {})
            if not any((field.linkage, key in includes)):
                del relationships[key]['data']

        return relationships

    def get_rtype(self):
        """ Return the string resource type as referenced by JSON API """

        # pylint: disable=no-member
        meta = getattr(self, 'JsonApiMeta', None)
        rtype = getattr(meta, 'rtype', None)
        if not rtype:
            msg = '"%s" must either have a `JsonApiMeta.rtype` attribute ' \
                  'or override `get_rtype()`' % self.__class__.__name__
            raise ImproperlyConfigured(msg)
        return rtype

    def is_valid(self, **kwargs):
        """ DRF override for error handling """

        try:
            return super(JsonApiSerializer, self).is_valid(**kwargs)
        except exceptions.ValidationError as exc:
            self.process_validation_errors(exc)

    def process_validation_errors(self, exc):
        """ This should be called when handling serializer exceptions

        Turn each DRF ValidationError error message in the
        `detail` property into an individual drfjsonapi
        ValidationError object. We want one error message
        per object.

        The JSON API spec has a `source` member that follows
        the JSON Pointer guidelines of RFC 6901 which points
        to a path in the payload representing the field
        responsible for the error.

        The renderer will then properly construct the source
        path. We do this so the source is meaningful across
        different renderers.
        """

        def _flatten(init, lkey=''):
            """ Recursively flatten embedded data types

            For embedded field types like a JSONField go through
            & convert the error key to a '/' separated string
            flattening them into a single dict.

            This allows for proper pointers in the error object
            """

            ret = {}
            for rkey, val in init.items():
                key = lkey + rkey
                if isinstance(val, dict):
                    ret.update(_flatten(val, key + '/'))
                else:
                    ret[key] = val
            return ret

        excs = []
        if isinstance(exc.detail, list):
            for error in exc.detail:
                excs.append(ResourceError(error))
        else:
            # prune the dict of all related field errors
            for field in list(exc.detail):
                for error in exc.detail[field]:
                    if field in self.related_fields:
                        excs.append(RelationshipError(field, error))
                        del exc.detail[field]

            # only field errors left now
            for field, errors in _flatten(exc.detail).items():
                for error in errors:
                    excs.append(FieldError(field, error))

        raise ManyExceptions(excs)

    def to_internal_value(self, data):
        """ DRF override for error handling

        Per the spec, first ensure the resource type provided
        matches that which is expected by this serializer.
        """

        if data['type'] != self.get_rtype():
            raise RtypeConflict(given=data['type'], rtype=self.get_rtype())

        try:
            return super(JsonApiSerializer, self).to_internal_value(data)
        except exceptions.ValidationError as exc:
            self.process_validation_errors(exc)

    def to_representation_sparse(self):
        """ Trim fields based on the sparse fields requested

        The JSON API spec uses the resource type (rtype) to
        qualify which fields should be returned.
        """

        try:
            sparse = self.context['request']._sparse_cache
            fields = sparse[self.get_rtype()]
        except (AttributeError, KeyError):
            return

        for key in self.fields.keys():
            if key not in fields:
                del self.fields[key]

    def to_representation(self, instance):
        """ Return an individual "Resource Object" object

        This should return a dict that is compliant with the
        "Resource Object" section of the JSON API spec. It
        will later be wrapped by the "Top Level" members.

        :spec:
            jsonapi.org/format/#document-resource-objects
        """

        self.to_representation_sparse()

        print('related query still occurs even if linkage=False included')
        data = super(JsonApiSerializer, self).to_representation(instance)
        return {
            'attributes': data,
            'links': self.get_links(instance),
            'meta': self.get_meta(),
            'relationships': self.get_relationships(data),
            'type': self.get_rtype(),
            'id': data.pop('id'),
        }

    def validate_embedded(self, data, fields):
        """ Validate embedded lists helper method

        This is like DRF's `validate()` except it takes a tuple
        of tuples in the following format:

            (field_name, serializer_class)

        The fields values will be iterated & validated individually
        using the provided serializer. It will then construct
        proper ValidationError's so the JSON pointers reference the
        list index value where the error occurred.

        This is mostly useful for embedded objects where the django
        field is a JSONField.
        """

        errors = {}

        for name, serializer_class in fields:
            value = data.get(name, [])  # empty if PATCH

            for idx, data_item in enumerate(value):
                serializer = serializer_class(data=data_item)
                if not serializer.is_valid():
                    for item_field, error in serializer.errors.items():
                        errors['%s/%s/%s' % (name, idx, item_field)] = error

        if errors:
            raise serializers.ValidationError(errors)

class JsonApiModelSerializer(JsonApiSerializer, serializers.ModelSerializer):
    """ JSON API ModelSerializer """

    pass

class PolymorphicModelSerializer(JsonApiModelSerializer):
    """ To be used with a JsonApiModelSerializer

    To use this, each model must have a designated field
    with a value that can be relied upon for distringuishing
    which serializer to use. The values MUST also be the
    JSON API "resource type" also known as rtype for the
    underlying model.

    The model field is specified by a `polymorphic_model_field`
    property on the JsonApiMeta object:

    ```
    class JsonApiMeta:
        rtype = 'books'
        polymorphic_model_field = 'rtype'
        polymorphic_mapping = {
            'paperback': PaperbackSerializer,
            'hardcover': HardcoverSerializer,
        }
    ```

    The `polymorphic_mapping` property is a dict containing
    the actual resource type string & serializer value which
    acts as a lookup table.
    """

    def __new__(cls, *args, **kwargs):
        """ DRF serializers override this too

        DRF passes in a single instance or a queryset as arg[0]
        and/or a data dict submitted by the requestor which is
        kwarg['data'].

        If we have data then always use it's type cause the
        requestor could be updating from one resource type to
        another. Otherwise fallback on the value of the models
        `polymorphic_model_field` value.
        """

        meta = getattr(cls, 'JsonApiMeta', None)
        model_field = getattr(meta, 'polymorphic_model_field', None)
        mapping = getattr(meta, 'polymorphic_mapping', {})

        if not model_field:
            msg = 'Using "%s" requires a "polymorphic_model_field" ' \
                  'property on the JsonApiMeta object'
            raise ImproperlyConfigured(msg % cls.__name__)

        if not mapping or not isinstance(mapping, dict):
            msg = 'Using "%s" requires a "polymorphic_mapping" ' \
                  'field which must be a dict on the JsonApiMeta object'
            raise ImproperlyConfigured(msg % cls.__name__)

        if kwargs.get('many'):
            return super().__new__(cls, *args, **kwargs)
        elif kwargs.get('data'):
            rtype = kwargs['data'].get('type')
        elif args and hasattr(args[0], model_field):
            rtype = getattr(args[0], model_field, None)
        else:
            return super().__new__(cls, *args, **kwargs)

        serializer = cls.get_polymorphic_serializer(mapping, rtype)
        return serializer(*args, **kwargs)

    @staticmethod
    def get_polymorphic_serializer(mapping, rtype):
        """ Find the serializer via rtype in the mapping """

        serializer = mapping.get(rtype)
        if not serializer:
            str_rtypes = ', '.join(mapping.keys())
            raise RtypeConflict(given=rtype, rtype=str_rtypes)
        return serializer

class ChildProxy:
    def __init__(self, list_serializer):
        self.list_serializer = list_serializer

    def __getattr__(self, name):
        if hasattr(self, name):
            return super().__getattr__(name)
        else:
            self.func_name = name
            return self

    def __call__(self, item):
        # item could be a models instance or data dict
        #rtype = getattr(item, <field name>, item.get('type'))

        # loop through children & find the right one
        for child in self.children:
            pass
        return 'XXX HERE', self.func_name, p, kw

class PolymorphicListSerializer(serializers.ListSerializer):
    # child won't be the PolymorphicModelSerializer which because
    # init calls instantiates the serializer again & since no
    # `many` cause this init pops it off then it will be one of
    # that actual serializers.
    #
    # that's not good

    #def __init__(self, *args, **kwargs):
    #    self.children = kwargs.pop('children')
    #    super().__init__(*args, **kwargs)
    #    for child in self.children:
    #        child.bind(field_name='', parent=self)

    #def __init__(self, *args, **kwargs):
    #    super().__init__(*args, **kwargs)
    #    self._child = self.child
    #    self.child = ChildProxy(list_serializer=self)
    pass
<commit_msg>added a new PolymorphicSerializer<commit_after>"""
    drfjsonapi.serializers
    ~~~~~~~~~~~~~~~~~~~~~~~

    DRF serializers to assist with a JSON API spec compliant API.

    Specifically, the JsonApiRenderer requires the use of the
    JsonApiSerializer
"""

from django.core.exceptions import ImproperlyConfigured
from django.utils.functional import cached_property
from rest_framework import exceptions
from rest_framework import serializers
from rest_framework.relations import ManyRelatedField
from .exceptions import (
    FieldError,
    ManyExceptions,
    RelationshipError,
    ResourceError,
    RtypeConflict,
)
from .relations import ResourceRelatedField
from .utils import _get_resource_url

# pylint: disable=abstract-method
class JsonApiSerializer(serializers.Serializer):
    """ JSON API Serializer """

    @property
    def related_fields(self):
        """ Return `self.fields` but limited to relationship fields """

        fields = {}
        for name, field in self.fields.items():
            if field.write_only:
                continue
            elif isinstance(field, ManyRelatedField):
                fields[name] = field.child_relation
            elif isinstance(field, ResourceRelatedField):
                fields[name] = field
        return fields

    def get_filterable_fields(self):
        """ Return a dict of fields allowed to be filtered on

        By default the `JsonApiMeta.filterable_fields` property
        is used to source the items.

        The key of each item is the string name of the field &
        the value is the `FilterField` object instance.
        """

        try:
            return self.JsonApiMeta.filterable_fields
        except AttributeError:
            return {}

    def get_default_include_fields(self):
        """ Return the default related fields to include

        This is only used if the requestor did not explicitly
        request includes per the JSON API spec.
        """

        try:
            return {
                k: v for k, v in self.get_includable_fields.items()
                if k in self.JsonApiMeta.default_include_fields
            }
        except (AttributeError, TypeError):
            return {}

    def get_includable_fields(self):
        """ Return includable related fields """

        try:
            return self.JsonApiMeta.includable_fields
        except AttributeError:
            return {}

    def get_links(self, instance):
        """ Return the "Links" object for an individual resource

        This should return a dict that is compliant with the
        document links section of the JSON API spec. Specifically,
        the links section of an individual "Resource Object".

        :spec:
            jsonapi.org/format/#document-links
        """

        rtype = self.get_rtype()
        resource_url = _get_resource_url(rtype, instance.id, self.context)
        return {'self': resource_url}

    def get_meta(self):
        """ Return the "Meta" object for an individual resource

        This should return a dict that is compliant with the
        document meta section of the JSON API spec. Specifically,
        the links section of an individual "Resource Object".

        :spec:
            jsonapi.org/format/#document-meta
        """

        return {}

    def get_relationships(self, data):
        """ Return the "Relationships Object" for a resource

        This should return a dict that is compliant with the
        "Relationships Object" section of the JSON API spec.
        Specifically, the contents of the top level `relationships`
        member of an individual resource boject.

        Relationships always get top-level links & meta objects
        but if they have been included or require "Linkage" then
        the "Resource Linkage" is added to the object as well.

        Often to-one relationships will have linkage=True while
        to-many's won't since there could be alot of them.

        NOTE: if the field is included then we don't worry
              about it here because the IncludesFilter will
              process all those defaults & populate the context's
              `includes` keys.

        :spec:
            jsonapi.org/format/#document-resource-object-relationships
        """

        relationships = {}
        for key, field in self.related_fields.items():
            relationships[key] = {
                'data': data.pop(key),
                'links': field.get_links(data['id']),
                'meta': field.get_meta(),
            }

            includes = self.context.get('includes', {})
            if not any((field.linkage, key in includes)):
                del relationships[key]['data']

        return relationships

    def get_rtype(self):
        """ Return the string resource type as referenced by JSON API """

        # pylint: disable=no-member
        meta = getattr(self, 'JsonApiMeta', None)
        rtype = getattr(meta, 'rtype', None)
        if not rtype:
            msg = '"%s" must either have a `JsonApiMeta.rtype` attribute ' \
                  'or override `get_rtype()`' % self.__class__.__name__
            raise ImproperlyConfigured(msg)
        return rtype

    def is_valid(self, **kwargs):
        """ DRF override for error handling """

        try:
            return super(JsonApiSerializer, self).is_valid(**kwargs)
        except exceptions.ValidationError as exc:
            self.process_validation_errors(exc)

    def process_validation_errors(self, exc):
        """ This should be called when handling serializer exceptions

        Turn each DRF ValidationError error message in the
        `detail` property into an individual drfjsonapi
        ValidationError object. We want one error message
        per object.

        The JSON API spec has a `source` member that follows
        the JSON Pointer guidelines of RFC 6901 which points
        to a path in the payload representing the field
        responsible for the error.

        The renderer will then properly construct the source
        path. We do this so the source is meaningful across
        different renderers.
        """

        def _flatten(init, lkey=''):
            """ Recursively flatten embedded data types

            For embedded field types like a JSONField go through
            & convert the error key to a '/' separated string
            flattening them into a single dict.

            This allows for proper pointers in the error object
            """

            ret = {}
            for rkey, val in init.items():
                key = lkey + rkey
                if isinstance(val, dict):
                    ret.update(_flatten(val, key + '/'))
                else:
                    ret[key] = val
            return ret

        excs = []
        if isinstance(exc.detail, list):
            for error in exc.detail:
                excs.append(ResourceError(error))
        else:
            # prune the dict of all related field errors
            for field in list(exc.detail):
                for error in exc.detail[field]:
                    if field in self.related_fields:
                        excs.append(RelationshipError(field, error))
                        del exc.detail[field]

            # only field errors left now
            for field, errors in _flatten(exc.detail).items():
                for error in errors:
                    excs.append(FieldError(field, error))

        raise ManyExceptions(excs)

    def to_internal_value(self, data):
        """ DRF override for error handling

        Per the spec, first ensure the resource type provided
        matches that which is expected by this serializer.
        """

        if data['type'] != self.get_rtype():
            raise RtypeConflict(given=data['type'], rtype=self.get_rtype())

        try:
            return super(JsonApiSerializer, self).to_internal_value(data)
        except exceptions.ValidationError as exc:
            self.process_validation_errors(exc)

    def to_representation_sparse(self):
        """ Trim fields based on the sparse fields requested

        The JSON API spec uses the resource type (rtype) to
        qualify which fields should be returned.
        """

        try:
            sparse = self.context['request']._sparse_cache
            fields = sparse[self.get_rtype()]
        except (AttributeError, KeyError):
            return

        for key in self.fields.keys():
            if key not in fields:
                del self.fields[key]

    def to_representation(self, instance):
        """ Return an individual "Resource Object" object

        This should return a dict that is compliant with the
        "Resource Object" section of the JSON API spec. It
        will later be wrapped by the "Top Level" members.

        :spec:
            jsonapi.org/format/#document-resource-objects
        """

        self.to_representation_sparse()

        print('related query still occurs even if linkage=False included')
        data = super(JsonApiSerializer, self).to_representation(instance)
        return {
            'attributes': data,
            'links': self.get_links(instance),
            'meta': self.get_meta(),
            'relationships': self.get_relationships(data),
            'type': self.get_rtype(),
            'id': data.pop('id'),
        }

    def validate_embedded(self, data, fields):
        """ Validate embedded lists helper method

        This is like DRF's `validate()` except it takes a tuple
        of tuples in the following format:

            (field_name, serializer_class)

        The fields values will be iterated & validated individually
        using the provided serializer. It will then construct
        proper ValidationError's so the JSON pointers reference the
        list index value where the error occurred.

        This is mostly useful for embedded objects where the django
        field is a JSONField.
        """

        errors = {}

        for name, serializer_class in fields:
            value = data.get(name, [])  # empty if PATCH

            for idx, data_item in enumerate(value):
                serializer = serializer_class(data=data_item)
                if not serializer.is_valid():
                    for item_field, error in serializer.errors.items():
                        errors['%s/%s/%s' % (name, idx, item_field)] = error

        if errors:
            raise serializers.ValidationError(errors)

class JsonApiModelSerializer(JsonApiSerializer, serializers.ModelSerializer):
    """ JSON API ModelSerializer """

    pass

class PolymorphicSerializer(JsonApiSerializer):
    """ A very basic Polymorphic serializer

    To use this each instance, typically a Django model,
    must have a designated field with a value that can be
    relied upon for distringuishing which serializer to use.
    The value MUST also be the JSON API "resource type" also
    known as rtype for the underlying instance.

    The instance field is specified by a `polymorphic_instance_field`
    property on the JsonApiMeta object:

    ```
    class JsonApiMeta:
        polymorphic_instance_field = 'species'
        polymorphic_serializers = (
            CatSerializer,
            DogSerializer,
        )
    ```

    The `polymorphic_serializers` property is an iterable
    containing all of the serializers that may be backing
    the instances & data.

    Implementation Details
    ~~~~~~~~~~~~~~~~~~~~~~

    The approach is similar to DRF's ListSerializer where
    native serializer methods are "proxied" to the underlying
    serializer.

    That means this should override every method or property
    the ListSerializer calls against the `child` attribute.
    """

    def __init__(self, *args, **kwargs):
        """ Ensure the required properties are present """

        super().__init__(*args, **kwargs)
        instance_field = self.get_polymorphic_instance_field()
        serializers = self.polymorphic_serializers

        if not instance_field:
            msg = 'Using "%s" requires a "polymorphic_instance_field" ' \
                  'property on the JsonApiMeta object'
            raise ImproperlyConfigured(msg % self.__name__)

        if not serializers or not isinstance(serializers, (list, tuple)):
            msg = 'Using "%s" requires a "polymorphic_serializers" ' \
                  'field which must be a dict on the JsonApiMeta object'
            raise ImproperlyConfigured(msg % self.__name__)

    @cached_property
    def polymorphic_serializers(self):
        """ Cached serializer instances of all backend serializer """

        return self.get_polymorphic_serializers()

    def get_polymorphic_instance_field(self):
        """ Return the string instance field name """

        meta = getattr(self, 'JsonApiMeta', None)
        return getattr(meta, 'polymorphic_instance_field', None)

    def get_polymorphic_serializers(self):
        """ Initialize all the backing serializers & return them """

        meta = getattr(self, 'JsonApiMeta', None)
        serializers = getattr(meta, 'polymorphic_serializers', ())
        return [s(context=self.context) for s in serializers]

    def get_polymorphic_serializer(self, item):
        """ Given an instance or dict find the right serializer

        Always presume it's a dict first since the caller
        could be changing the type in an update. Fallback
        to looking at the instance last.
        """

        instance_field = self.get_polymorphic_instance_field()
        try:
            rtype = item.get('type', item.get(instance_field))
        except AttributeError:
            rtype = getattr(item, instance_field)
        for serializer in self.polymorphic_serializers:
            if serializer.get_rtype() == rtype:
                return serializer

    def create(self, validated_data):
        """ DRF override, no type key present instead use instance_field """

        serializer = self.get_polymorphic_serializer(validated_data)
        return serializer.create(validated_data)

    def update(self, instance, validated_data):
        """ DRF override, no type key present instead use instance_field """

        serializer = self.get_polymorphic_serializer(validated_data)
        return serializer.update(instance, validated_data)

    def run_validation(self, data):
        """ DRF override, use the type key """

        serializer = self.get_polymorphic_serializer(data)
        return serializer.run_validation(data)

    def to_representation(self, data):
        """ DRF override, use the instance_field on the instance """

        serializer = self.get_polymorphic_serializer(data)
        return serializer.to_representation(data)
<|endoftext|>


<commit_before>import re
from cameo.data import metanetx
from cameo.core.reaction import Reaction
from cameo.core.metabolite import Metabolite
import logging
logger = logging.getLogger(__name__)

def clean_bigg_id(string):
    return re.sub(r"bigg:|dsh", "", string)

def get_existing_metabolite(mnx_id, model, compartment):
    """Find compartment in the model by Metanetx id.

    Parameters
    ----------
    mnx_id : string
        Metanetx id
    model
        cobra model
    compartment : string
        f.e "_c"

    Returns
    -------
    Metabolite or None

    """
    if not mnx_id:
        return
    try:
        clean_id = clean_bigg_id(metanetx.mnx2bigg[mnx_id])
        return model.metabolites.get_by_id(clean_id + compartment)
    except KeyError:
        try:
            return model.metabolites.get_by_id(mnx_id + compartment)
        except KeyError:
            pass

def contains_carbon(metabolite):  # TODO: use method from Metabolite class when this change is merged
    if not metabolite.formula:
        raise ValueError("No formula for metabolite {}, it's unknown if there is carbon in it")
    return 'C' in metabolite.elements

def find_metabolite_info(met_id):
    """Find chemical formula of metabolite in metanetx.chem_prop dictionary

    Parameters
    ----------
    met_id : string
        string of format "<metabolite_id>_<compartment_id>", where <metabolite_id> is a BIGG id or a Metanetx id

    Returns
    -------
    pandas row or None

    """
    met_id = met_id[:-2]
    try:
        if met_id in metanetx.chem_prop.index:
            return metanetx.chem_prop.loc[met_id]
        return metanetx.chem_prop.loc[metanetx.all2mnx['bigg:' + met_id]]
    except KeyError:
        return None

class ModelModificationMixin(object):
    """
    Base model modification class, providing methods for adding adapter and exchange reactions for new metabolites
    """
    model = None
    added_reactions = None

    def create_exchange(self, metabolite):
        """For given metabolite A_c from c compartment, create:
        a) corresponding metabolite A_e from e compartment;
        b) adapter reaction A_c <--> A_e
        c) exchange reaction A_e -->

        Parameters
        ----------
        metabolite : Metabolite
            metabolite id in format <bigg_id>_c, f.e. Nacsertn_c

        Returns
        -------

        """
        exchange_metabolite = Metabolite(metabolite.id.replace('_c', '_e'), formula=metabolite.formula, compartment='e')
        self.add_adapter_reaction(metabolite, exchange_metabolite)
        self.add_exchange_reaction(exchange_metabolite)

    def add_exchange_reaction(self, metabolite):
        """Add exchange reaction "A --> " for given metabolite A
         If reaction exists, log and pass

        Parameters
        ----------
        metabolite : basestring
            metabolite id in format <bigg_id>_<compartment_id>, f.e. Nacsertn_c

        Returns
        -------

        """
        try:
            logger.debug('Add exchange reaction for metabolite: {}'.format(metabolite.id))
            exchange_reaction = self.model.add_exchange(metabolite, prefix='EX_')
            self.added_reactions.add(exchange_reaction.id)
        except ValueError:
            logger.debug('Exchange reaction exists for metabolite {}'.format(metabolite.id))

    def add_adapter_reaction(self, metabolite, existing_metabolite):
        """Add adapter reaction A <--> B for metabolites A and B

        Parameters
        ----------
        metabolite : Metabolite
            metabolite A
        existing_metabolite : Metabolite
            metabolite B

        Returns
        -------

        """
        try:
            adapter_reaction = Reaction(str('adapter_' + metabolite.id + '_' + existing_metabolite.id))
            adapter_reaction.lower_bound = -1000
            adapter_reaction.add_metabolites({metabolite: -1, existing_metabolite: 1})
            self.model.add_reactions([adapter_reaction])
            self.added_reactions.add(adapter_reaction.id)
            logger.debug('Adapter reaction added: {} <--> {}'.format(metabolite.id, existing_metabolite.id))
        except Exception:  # TODO: raise a reasonable exception on cameo side if the reaction exists
            logger.debug('Adapter reaction exists: {} <--> {}'.format(metabolite.id, existing_metabolite.id))

    def add_demand_reaction(self, metabolite):
        """For metabolite in e compartment with existing exchange reaction, make it possible to consume metabolite
        by decreasing the lower bound of exchange reaction

        Parameters
        ----------
        metabolite : Metabolite
            metabolite from e compartment, f.e. melatn_e

        Returns
        -------

        """
        exchange_reaction = list(set(metabolite.reactions).intersection(self.model.exchanges))[0]
        if exchange_reaction.lower_bound >= 0:
            exchange_reaction.lower_bound = -1 if contains_carbon(metabolite) else -1000

    @staticmethod
    def annotate_new_metabolite(metabolite):
        """Find information about new metabolite in chem_prop dictionary and add it to model

        Parameters
        ----------
        metabolite : Metabolite
            new metabolite

        Returns
        -------

        """
        info = find_metabolite_info(metabolite.id)
        if info is not None:
            metabolite.formula = info['formula']
            metabolite.name = info['name']
            metabolite.annotation = info.to_dict()
        else:
            logger.debug('No formula for {}'.format(metabolite.id))
<commit_msg>fix: model metabolite<commit_after>import re
from cameo.data import metanetx
from cameo.core.reaction import Reaction
from cameo.core.metabolite import Metabolite
import logging
logger = logging.getLogger(__name__)

def clean_bigg_id(string):
    return re.sub(r"bigg:|dsh", "", string)

def get_existing_metabolite(mnx_id, model, compartment):
    """Find compartment in the model by Metanetx id.

    Parameters
    ----------
    mnx_id : string
        Metanetx id
    model
        cobra model
    compartment : string
        f.e "_c"

    Returns
    -------
    Metabolite or None

    """
    if not mnx_id:
        return
    try:
        clean_id = clean_bigg_id(metanetx.mnx2bigg[mnx_id])
        return model.metabolites.get_by_id(clean_id + compartment)
    except KeyError:
        try:
            return model.metabolites.get_by_id(mnx_id + compartment)
        except KeyError:
            pass

def contains_carbon(metabolite):  # TODO: use method from Metabolite class when this change is merged
    if not metabolite.formula:
        raise ValueError("No formula for metabolite {}, it's unknown if there is carbon in it")
    return 'C' in metabolite.elements

def find_metabolite_info(met_id):
    """Find chemical formula of metabolite in metanetx.chem_prop dictionary

    Parameters
    ----------
    met_id : string
        string of format "<metabolite_id>_<compartment_id>", where <metabolite_id> is a BIGG id or a Metanetx id

    Returns
    -------
    pandas row or None

    """
    met_id = met_id[:-2]
    try:
        if met_id in metanetx.chem_prop.index:
            return metanetx.chem_prop.loc[met_id]
        return metanetx.chem_prop.loc[metanetx.all2mnx['bigg:' + met_id]]
    except KeyError:
        return None

class ModelModificationMixin(object):
    """
    Base model modification class, providing methods for adding adapter and exchange reactions for new metabolites
    """
    model = None
    added_reactions = None

    def create_exchange(self, metabolite):
        """For given metabolite A_c from c compartment, create:
        a) corresponding metabolite A_e from e compartment;
        b) adapter reaction A_c <--> A_e
        c) exchange reaction A_e -->

        Parameters
        ----------
        metabolite : Metabolite
            metabolite id in format <bigg_id>_c, f.e. Nacsertn_c

        Returns
        -------

        """
        exchange_metabolite = Metabolite(metabolite.id.replace('_c', '_e'), formula=metabolite.formula, compartment='e')
        self.add_adapter_reaction(metabolite, exchange_metabolite)
        self.add_exchange_reaction(exchange_metabolite)

    def add_exchange_reaction(self, metabolite):
        """Add exchange reaction "A --> " for given metabolite A
         If reaction exists, log and pass

        Parameters
        ----------
        metabolite : basestring
            metabolite id in format <bigg_id>_<compartment_id>, f.e. Nacsertn_c

        Returns
        -------

        """
        try:
            logger.debug('Add exchange reaction for metabolite: {}'.format(metabolite.id))
            exchange_reaction = self.model.add_exchange(metabolite, prefix='EX_')
            self.added_reactions.add(exchange_reaction.id)
        except ValueError:
            logger.debug('Exchange reaction exists for metabolite {}'.format(metabolite.id))

    def add_adapter_reaction(self, metabolite, existing_metabolite):
        """Add adapter reaction A <--> B for metabolites A and B

        Parameters
        ----------
        metabolite : Metabolite
            metabolite A
        existing_metabolite : Metabolite
            metabolite B

        Returns
        -------

        """
        try:
            adapter_reaction = Reaction(str('adapter_' + metabolite.id + '_' + existing_metabolite.id))
            adapter_reaction.lower_bound = -1000
            adapter_reaction.add_metabolites({metabolite: -1, existing_metabolite: 1})
            self.model.add_reactions([adapter_reaction])
            self.added_reactions.add(adapter_reaction.id)
            logger.debug('Adapter reaction added: {} <--> {}'.format(metabolite.id, existing_metabolite.id))
        except Exception:  # TODO: raise a reasonable exception on cameo side if the reaction exists
            logger.debug('Adapter reaction exists: {} <--> {}'.format(metabolite.id, existing_metabolite.id))

    def add_demand_reaction(self, metabolite):
        """For metabolite in e compartment with existing exchange reaction, make it possible to consume metabolite
        by decreasing the lower bound of exchange reaction

        Parameters
        ----------
        metabolite : Metabolite
            metabolite from e compartment, f.e. melatn_e

        Returns
        -------

        """
        exchange_reaction = list(set(metabolite.reactions).intersection(self.model.exchanges))[0]
        if exchange_reaction.lower_bound >= 0:
            exchange_reaction.lower_bound = -1 if contains_carbon(metabolite) else -1000

    @staticmethod
    def annotate_new_metabolite(metabolite):
        """Find information about new metabolite in chem_prop dictionary and add it to model

        Parameters
        ----------
        metabolite : Metabolite
            new metabolite

        Returns
        -------

        """
        info = find_metabolite_info(metabolite.id)
        if info is not None:
            metabolite.formula = info['formula']
            metabolite.name = info['name']
            metabolite.annotation = info.to_dict()
        else:
            logger.debug('No formula for {}'.format(metabolite.id))

    def model_metabolite(self, entity, compartment='_e'):
        """Get metabolite associated with this model for a given entity

        Parameters
        ----------
        entity
            a chemical entity as provided by iloop
        compartment
            the compartment where to find the metabolite, e.g. _e for exchange
        Returns
        -------
        the model metabolite (or None if no matching found)
        """
        mnx_id = metanetx.all2mnx.get('chebi:' + str(entity.chebi_id))
        return get_existing_metabolite(mnx_id, self.model, compartment)
<|endoftext|>


<commit_before># This file is part of QuTiP: Quantum Toolbox in Python.
#
#    Copyright (c) 2011 and later, Paul D. Nation and Robert J. Johansson.
#    All rights reserved.
#
#    Redistribution and use in source and binary forms, with or without
#    modification, are permitted provided that the following conditions are
#    met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#    3. Neither the name of the QuTiP: Quantum Toolbox in Python nor the names
#       of its contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
#    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
#    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
###############################################################################
import numpy as np
from numpy.testing import assert_, run_module_suite, assert_equal
import scipy.sparse as sp

from qutip.random_objects import rand_dm
from qutip.operators import create, destroy, qeye
from qutip.states import coherent
from qutip.sparse import sp_bandwidth, sp_permute, sp_reverse_permute
from qutip.graph import reverse_cuthill_mckee

def _permutateIndexes(array, row_perm, col_perm):
    return array[np.ix_(row_perm, col_perm)]

def test_sparse_symmetric_permute():
    "Sparse: Symmetric Permute"
    # CSR version
    A = rand_dm(25, 0.5)
    perm = np.random.permutation(25)
    x = sp_permute(A.data, perm, perm).toarray()
    z = _permutateIndexes(A.full(), perm, perm)
    assert_equal((x - z).all(), 0)
    # CSC version
    B = A.data.tocsc()
    y = sp_permute(B, perm, perm).toarray()
    assert_equal((y - z).all(), 0)

def test_sparse_nonsymmetric_permute():
    "Sparse: Nonsymmetric Permute"
    # CSR version
    A = rand_dm(25, 0.5)
    rperm = np.random.permutation(25)
    cperm = np.random.permutation(25)
    x = sp_permute(A.data, rperm, cperm).toarray()
    z = _permutateIndexes(A.full(), rperm, cperm)
    assert_equal((x - z).all(), 0)
    # CSC version
    B = A.data.tocsc()
    y = sp_permute(B, rperm, cperm).toarray()
    assert_equal((y - z).all(), 0)

def test_sparse_symmetric_reverse_permute():
    "Sparse: Symmetric Reverse Permute"
    # CSR version
    A = rand_dm(25, 0.5)
    perm = np.random.permutation(25)
    x = sp_permute(A.data, perm, perm)
    B = sp_reverse_permute(x, perm, perm)
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSC version
    B = A.data.tocsc()
    perm = np.random.permutation(25)
    x = sp_permute(B, perm, perm)
    B = sp_reverse_permute(x, perm, perm)
    assert_equal((A.full() - B.toarray()).all(), 0)

def test_sparse_nonsymmetric_reverse_permute():
    "Sparse: Nonsymmetric Reverse Permute"
    # CSR square array check
    A = rand_dm(25, 0.5)
    rperm = np.random.permutation(25)
    cperm = np.random.permutation(25)
    x = sp_permute(A.data, rperm, cperm)
    B = sp_reverse_permute(x, rperm, cperm)
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSC square array check
    A = rand_dm(25, 0.5)
    rperm = np.random.permutation(25)
    cperm = np.random.permutation(25)
    B = A.data.tocsc()
    x = sp_permute(B, rperm, cperm)
    B = sp_reverse_permute(x, rperm, cperm)
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSR column vector check
    A = coherent(25, 1)
    rperm = np.random.permutation(25)
    x = sp_permute(A.data, rperm, [])
    B = sp_reverse_permute(x, rperm, [])
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSC column vector check
    A = coherent(25, 1)
    rperm = np.random.permutation(25)
    B = A.data.tocsc()
    x = sp_permute(B, rperm, [])
    B = sp_reverse_permute(x, rperm, [])
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSR row vector check
    A = coherent(25, 1).dag()
    cperm = np.random.permutation(25)
    x = sp_permute(A.data, [], cperm)
    B = sp_reverse_permute(x, [], cperm)
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSC row vector check
    A = coherent(25, 1).dag()
    cperm = np.random.permutation(25)
    B = A.data.tocsc()
    x = sp_permute(B, [], cperm)
    B = sp_reverse_permute(x, [], cperm)
    assert_equal((A.full() - B.toarray()).all(), 0)

def test_sp_bandwidth():
    "Sparse: Bandwidth"
    A = sp.rand(10,10,density=0.3,format='csr')
    ans1 = sp_bandwidth(A)
    A = A.toarray()
    i,j = np.nonzero(A)
    ans2 = ((j-i).max()+(i-j).max()+1,(i-j).max(),(j-i).max())
    assert_equal(ans1, ans2)

if __name__ == "__main__":
    run_module_suite()
<commit_msg>Added randomized sp_profile tests<commit_after># This file is part of QuTiP: Quantum Toolbox in Python.
#
#    Copyright (c) 2011 and later, Paul D. Nation and Robert J. Johansson.
#    All rights reserved.
#
#    Redistribution and use in source and binary forms, with or without
#    modification, are permitted provided that the following conditions are
#    met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#    3. Neither the name of the QuTiP: Quantum Toolbox in Python nor the names
#       of its contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
#    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
#    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
###############################################################################
import numpy as np
from numpy.testing import assert_, run_module_suite, assert_equal
import scipy.sparse as sp

from qutip.random_objects import rand_dm
from qutip.operators import create, destroy, qeye
from qutip.states import coherent
from qutip.sparse import (sp_bandwidth, sp_permute, sp_reverse_permute,
                        sp_profile)
from qutip.graph import reverse_cuthill_mckee

def _permutateIndexes(array, row_perm, col_perm):
    return array[np.ix_(row_perm, col_perm)]

def _dense_profile(B):
    row_pro = 0
    for i in range(B.shape[0]):
        j = np.where(B[i,:]!=0)[0]
        if np.any(j):
            if j[-1] > i:
                row_pro += (j[-1]-i)
    col_pro = 0
    for j in range(B.shape[0]):
        i = np.where(B[:,j]!=0)[0]
        if np.any(i):
            if i[-1] > j:
                col_pro += i[-1]-j
    ans = (row_pro+col_pro,col_pro,row_pro)
    return ans

def test_sparse_symmetric_permute():
    "Sparse: Symmetric Permute"
    # CSR version
    A = rand_dm(25, 0.5)
    perm = np.random.permutation(25)
    x = sp_permute(A.data, perm, perm).toarray()
    z = _permutateIndexes(A.full(), perm, perm)
    assert_equal((x - z).all(), 0)
    # CSC version
    B = A.data.tocsc()
    y = sp_permute(B, perm, perm).toarray()
    assert_equal((y - z).all(), 0)

def test_sparse_nonsymmetric_permute():
    "Sparse: Nonsymmetric Permute"
    # CSR version
    A = rand_dm(25, 0.5)
    rperm = np.random.permutation(25)
    cperm = np.random.permutation(25)
    x = sp_permute(A.data, rperm, cperm).toarray()
    z = _permutateIndexes(A.full(), rperm, cperm)
    assert_equal((x - z).all(), 0)
    # CSC version
    B = A.data.tocsc()
    y = sp_permute(B, rperm, cperm).toarray()
    assert_equal((y - z).all(), 0)

def test_sparse_symmetric_reverse_permute():
    "Sparse: Symmetric Reverse Permute"
    # CSR version
    A = rand_dm(25, 0.5)
    perm = np.random.permutation(25)
    x = sp_permute(A.data, perm, perm)
    B = sp_reverse_permute(x, perm, perm)
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSC version
    B = A.data.tocsc()
    perm = np.random.permutation(25)
    x = sp_permute(B, perm, perm)
    B = sp_reverse_permute(x, perm, perm)
    assert_equal((A.full() - B.toarray()).all(), 0)

def test_sparse_nonsymmetric_reverse_permute():
    "Sparse: Nonsymmetric Reverse Permute"
    # CSR square array check
    A = rand_dm(25, 0.5)
    rperm = np.random.permutation(25)
    cperm = np.random.permutation(25)
    x = sp_permute(A.data, rperm, cperm)
    B = sp_reverse_permute(x, rperm, cperm)
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSC square array check
    A = rand_dm(25, 0.5)
    rperm = np.random.permutation(25)
    cperm = np.random.permutation(25)
    B = A.data.tocsc()
    x = sp_permute(B, rperm, cperm)
    B = sp_reverse_permute(x, rperm, cperm)
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSR column vector check
    A = coherent(25, 1)
    rperm = np.random.permutation(25)
    x = sp_permute(A.data, rperm, [])
    B = sp_reverse_permute(x, rperm, [])
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSC column vector check
    A = coherent(25, 1)
    rperm = np.random.permutation(25)
    B = A.data.tocsc()
    x = sp_permute(B, rperm, [])
    B = sp_reverse_permute(x, rperm, [])
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSR row vector check
    A = coherent(25, 1).dag()
    cperm = np.random.permutation(25)
    x = sp_permute(A.data, [], cperm)
    B = sp_reverse_permute(x, [], cperm)
    assert_equal((A.full() - B.toarray()).all(), 0)
    # CSC row vector check
    A = coherent(25, 1).dag()
    cperm = np.random.permutation(25)
    B = A.data.tocsc()
    x = sp_permute(B, [], cperm)
    B = sp_reverse_permute(x, [], cperm)
    assert_equal((A.full() - B.toarray()).all(), 0)

def test_sp_bandwidth():
    "Sparse: Bandwidth"
    for kk in range(10):
        A = sp.rand(100,100,density=0.1,format='csr')
        ans1 = sp_bandwidth(A)
        A = A.toarray()
        i,j = np.nonzero(A)
        ans2 = ((j-i).max()+(i-j).max()+1,(i-j).max(),(j-i).max())
        assert_equal(ans1, ans2)

    for kk in range(10):
        A = sp.rand(100,100,density=0.1,format='csc')
        ans1 = sp_bandwidth(A)
        A = A.toarray()
        i,j = np.nonzero(A)
        ans2 = ((j-i).max()+(i-j).max()+1,(i-j).max(),(j-i).max())
        assert_equal(ans1, ans2)

def test_sp_profile():
    "Sparse: Profile"
    for kk in range(10):
        A = sp.rand(1000,1000,0.1,format='csr')
        pro = sp_profile(A)
        B = A.toarray()
        ans = _dense_profile(B)
        assert_equal(pro, ans)

    for kk in range(10):
        A = sp.rand(1000,1000,0.1,format='csc')
        pro = sp_profile(A)
        B = A.toarray()
        ans = _dense_profile(B)
        assert_equal(pro, ans)

if __name__ == "__main__":
    run_module_suite()
<|endoftext|>


<commit_before>import azurerm
import json

# Load Azure app defaults
try:
   with open('azurermconfig.json') as configFile:
      configData = json.load(configFile)
except FileNotFoundError:
   print("Error: Expecting vmssConfig.json in current folder")
   sys.exit()

tenant_id = configData['tenantId']
app_id = configData['appId']
app_secret = configData['appSecret']
subscription_id = configData['subscriptionId']

access_token = azurerm.get_access_token(tenant_id, app_id, app_secret)

# list the VMs
vmsslist = azurerm.list_vmss_sub(access_token, subscription_id)
print(json.dumps(vmsslist, sort_keys=False, indent=2, separators=(',', ': ')))

'''
for vm in vmsslist['value']:
    # print(json.dumps(vm, sort_keys=False, indent=2, separators=(',', ': ')))
    print(vm['name'])

# loop through resource groups
resource_groups = azurerm.list_resource_groups(access_token, subscription_id)
for rg in resource_groups["value"]:
    rgname = rg["name"]
    vmsslist = azurerm.list_vmss(access_token, subscription_id, rgname)
    for vmss in vmsslist['value']:
        name = vmss['name']
        location = vmss['location']
        capacity = vmss['sku']['capacity']
        offer = vmss['properties']['virtualMachineProfile']['storageProfile']['imageReference']['offer']
        sku = vmss['properties']['virtualMachineProfile']['storageProfile']['imageReference']['sku']
        print(''.join(['Name: ', name,
                       ', RG: ', rgname,
                       ', location: ', location,
                       ', Capacity: ', str(capacity),
                       ', OS: ', offer, ' ', sku]))
        print('Virtual machines...')
        vms = azurerm.list_vmss_vms(access_token, subscription_id, rgname, name)
        #print(json.dumps(vms, sort_keys=False, indent=2, separators=(',', ': ')))
        for vm in vms['value']:
            vmid = vm['instanceId']
            vmnic = azurerm.get_vmss_vm_nics(access_token, subscription_id, rgname, name, vmid)
            privateIP = vmnic['value'][0]['properties']['ipConfigurations'][0]['properties']['privateIPAddress']
            #print(json.dumps(vmnic, sort_keys=False, indent=2, separators=(',', ': ')))
            print(vmid + ', ' + vm['name'] + ', private IP:' + privateIP)

#startoutput = azurerm.start_vmss(access_token, subscription_id, 'guydock3', 'guydock3')
# print(startoutput)

# scaleoutput = azurerm.scale_vmss(access_token, subscription_id, 'guydock3', 'guydock3', 'Standard_A1', 'Standard', 5)
# print(scaleoutput)
'''

<commit_msg>pylint clean list_vmss.py<commit_after>'''list_vmss.py - list the scale sets in a subscription'''
import json
import sys

import azurerm

def main():
    '''main reoutine'''
    # Load Azure app defaults
    try:
        with open('azurermconfig.json') as config_file:
            config_data = json.load(config_file)
    except FileNotFoundError:
        sys.exit("Error: Expecting azurermconfig.json in current folder")

    tenant_id = config_data['tenantId']
    app_id = config_data['appId']
    app_secret = config_data['appSecret']
    subscription_id = config_data['subscriptionId']

    access_token = azurerm.get_access_token(tenant_id, app_id, app_secret)

    # list the VMs
    vmsslist = azurerm.list_vmss_sub(access_token, subscription_id)
    print(json.dumps(vmsslist, sort_keys=False, indent=2, separators=(',', ': ')))

    '''
    for vmss in vmsslist['value']:
        # print(json.dumps(vmss, sort_keys=False, indent=2, separators=(',', ': ')))
        print(vmss['name'])
    '''

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from django.conf import settings

def get_meta(request):
    return {
        'HTTP_X_FORWARDED_FOR': request.META.get('HTTP_X_FORWARDED_FOR'),
        'REMOTE_ADDR': request.META.get('REMOTE_ADDR'),
    }

def analytics_enabled_for_email(email_address):
    from corehq.apps.users.models import CouchUser
    user = CouchUser.get_by_username(email_address)
    return user.analytics_enabled if user else True

def get_instance_string():
    instance = settings.ANALYTICS_CONFIG.get('HQ_INSTANCE', '')
    env = '' if instance == 'www' else instance + '_'
    return env
<commit_msg>add utilities for blocking hubspot data<commit_after>from django.conf import settings

from corehq.apps.accounting.models import Subscription, BillingAccount

def get_meta(request):
    return {
        'HTTP_X_FORWARDED_FOR': request.META.get('HTTP_X_FORWARDED_FOR'),
        'REMOTE_ADDR': request.META.get('REMOTE_ADDR'),
    }

def analytics_enabled_for_email(email_address):
    from corehq.apps.users.models import CouchUser
    user = CouchUser.get_by_username(email_address)
    return user.analytics_enabled if user else True

def is_email_blocked_from_hubspot(email_address):
    email_domain = email_address.split('@')[-1]
    return BillingAccount.objects.filter(
        is_active=True,
        block_email_domains_from_hubspot__contains=[email_domain],
    ).exists()

def is_domain_blocked_from_hubspot(domain):
    return Subscription.visible_objects.filter(
        is_active=True,
        subscriber__domain=domain,
        account__is_active=True,
        account__block_hubspot_data_for_all_users=True,
    ).exists()

def hubspot_enabled_for_user(couch_user):
    if is_email_blocked_from_hubspot(couch_user.username):
        return False
    for domain in couch_user.get_domains():
        if is_domain_blocked_from_hubspot(domain):
            return False
    return couch_user.analytics_enabled

def hubspot_enabled_for_email(email_address):
    from corehq.apps.users.models import CouchUser
    user = CouchUser.get_by_username(email_address)
    return hubspot_enabled_for_user(user) if user else True

def get_blocked_hubspot_domains():
    return list(Subscription.visible_objects.filter(
        account__block_hubspot_data_for_all_users=True,
        is_active=True,
        account__is_active=True,
    ).values_list('subscriber__domain', flat=True))

def get_blocked_hubspot_email_domains():
    return [_email for email_list in BillingAccount.objects.filter(
        is_active=True,
    ).exclude(
        block_email_domains_from_hubspot=[],
    ).values_list(
        'block_email_domains_from_hubspot',
        flat=True,
    ) for _email in email_list]

def get_instance_string():
    instance = settings.ANALYTICS_CONFIG.get('HQ_INSTANCE', '')
    env = '' if instance == 'www' else instance + '_'
    return env

<|endoftext|>


<commit_before>import math
import time
import requests
import logging

from django.conf import settings

from corehq.util.metrics import metrics_counter
from corehq.apps.accounting.models import Subscription, BillingAccount
from corehq.apps.es.users import UserES
from corehq.apps.users.models import WebUser, CommCareUser

logger = logging.getLogger('analytics')

MAX_API_RETRIES = 5

ALLOWED_CONVERSIONS = [
    'Blog',
    'Case study/ Ev. Base',
    'Contact us',
    'Event',
    'Live Chat',
    'Newletter',
    'Newsletter',
    'Offer',
    'Outbound',
    'Paid Ads',
    'Video',
]

def get_meta(request):
    return {
        'HTTP_X_FORWARDED_FOR': request.META.get('HTTP_X_FORWARDED_FOR'),
        'REMOTE_ADDR': request.META.get('REMOTE_ADDR'),
    }

def analytics_enabled_for_email(email_address):
    from corehq.apps.users.models import CouchUser
    user = CouchUser.get_by_username(email_address)
    return user.analytics_enabled if user else True

def is_domain_blocked_from_hubspot(domain):
    return Subscription.visible_objects.filter(
        is_active=True,
        subscriber__domain=domain,
        account__is_active=True,
        account__block_hubspot_data_for_all_users=True,
    ).exists()

def hubspot_enabled_for_user(user):
    """
    Check if a user (or account policy) has given permission for Hubspot
    analytics to be synced for a given user.
    :param user: CouchUser or WebUser
    :return: Boolean (True if hubspot is enabled/allowed)
    """
    if isinstance(user, WebUser):
        web_user = user
    else:
        web_user = WebUser.get_by_username(user.username)
    if web_user:
        for domain in web_user.get_domains():
            if is_domain_blocked_from_hubspot(domain):
                return False
    else:
        commcare_user = CommCareUser.get_by_username(user.username)
        if is_domain_blocked_from_hubspot(commcare_user.domain):
            return False
    return user.analytics_enabled

def hubspot_enabled_for_email(email_address):
    from corehq.apps.users.models import CouchUser
    user = CouchUser.get_by_username(email_address)
    return hubspot_enabled_for_user(user) if user else True

def get_blocked_hubspot_domains():
    """
    Get the list of domains / project spaces that have active subscriptions
    with accounts that have blocked hubspot data.
    :return: list
    """
    return list(Subscription.visible_objects.filter(
        account__block_hubspot_data_for_all_users=True,
        is_active=True,
        account__is_active=True,
    ).values_list('subscriber__domain', flat=True))

def get_blocked_hubspot_accounts():
    return [
        f'{account[1]} - ID # {account[0]}'
        for account in BillingAccount.objects.filter(
            block_hubspot_data_for_all_users=True,
            is_active=True,
        ).values_list('id', 'name')
    ]

def get_instance_string():
    instance = settings.ANALYTICS_CONFIG.get('HQ_INSTANCE', '')
    env = '' if instance == 'www' else instance + '_'
    return env

def _delete_hubspot_contact(vid, retry_num=0):
    """
    Permanently deletes a Hubspot contact.
    :param vid:  (the contact ID)
    :param retry_num: the number of the current retry attempt
    :return: boolean if contact was deleted
    """
    if retry_num > 0:
        time.sleep(10)  # wait 10 seconds if this is another retry attempt

    api_key = settings.ANALYTICS_IDS.get('HUBSPOT_API_KEY', None)
    if api_key:
        try:
            req = requests.delete(
                f'https://api.hubapi.com/contacts/v1/contact/vid/{vid}',
                params={
                    'hapikey': api_key,
                }
            )
            if req.status_code == 404:
                return False
            req.raise_for_status()
        except (ConnectionError, requests.exceptions.HTTPError) as e:
            metrics_counter(
                'commcare.hubspot_data.retry.delete_hubspot_contact',
                tags={
                    'vid': vid,
                }
            )
            if retry_num <= MAX_API_RETRIES:
                return _delete_hubspot_contact(vid, retry_num + 1)
            else:
                logger.error(f"Failed to delete Hubspot contact {vid} due to "
                             f"{str(e)}.")
        else:
            return True
    return False

def _get_contact_ids_to_delete(list_of_emails, retry_num=0):
    """
    Gets a list of Contact IDs on Hubspot from a list of emails.
    If an email in the list doesn't exist on Hubspot, it's simply ignored.
    We also check the list returned from HubSpot to ensure that users
    who engaged with HubSpot under an allowed first conversion action are
    not removed from HubSpot.
    :param list_of_emails:
    :param retry_num: the number of the current retry attempt
    :return: list of contact ids
    """
    if retry_num > 0:
        time.sleep(10)  # wait 10 seconds if this is another retry attempt

    api_key = settings.ANALYTICS_IDS.get('HUBSPOT_API_KEY', None)
    if api_key:
        try:
            req = requests.get(
                "https://api.hubapi.com/contacts/v1/contact/emails/batch/",
                params={
                    'hapikey': api_key,
                    'email': list_of_emails,
                },
            )
            if req.status_code == 404:
                return []
            req.raise_for_status()
        except (ConnectionError, requests.exceptions.HTTPError) as e:
            metrics_counter(
                'commcare.hubspot_data.retry.get_contact_ids_for_emails',
                tags={
                    'emails': list_of_emails.join(', '),
                }
            )
            if retry_num <= MAX_API_RETRIES:
                return _get_contact_ids_to_delete(list_of_emails, retry_num + 1)
            else:
                logger.error(f"Failed to get Hubspot contact ids for emails "
                             f"{list_of_emails.join(', ')} due to {str(e)}.")
        else:
            ids_to_delete = []
            for contact_id, data in req.json().items():
                first_conversion_status = data.get(
                    'properties', {}
                ).get('first_conversion_clustered_', {}).get('value')

                # If a user's first conversion IS in the allowed list, it means
                # they have directly interacted with us and we want to keep them
                # in the email list. However, we will not send project data
                # about them to HubSpot as they will still be blocked from
                # updates in track_periodic_data.
                if first_conversion_status not in ALLOWED_CONVERSIONS:
                    ids_to_delete.append(contact_id)
            return ids_to_delete
    return []

def remove_blocked_domain_contacts_from_hubspot(stdout=None):
    """
    Removes contacts from Hubspot that are members of blocked domains.
    :param stdout: the stdout of a management command (if applicable)
    """
    blocked_domains = get_blocked_hubspot_domains()
    for domain in blocked_domains:
        if stdout:
            stdout.write(f"\n\nChecking DOMAIN {domain}...")
        user_query = UserES().domain(domain).source(['email', 'username'])

        total_users = user_query.count()
        chunk_size = 30  # Hubspot recommends fewer than 100 emails per request
        num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))

        for chunk in range(num_chunks):
            blocked_users = (user_query
                             .size(chunk_size)
                             .start(chunk * chunk_size)
                             .run()
                             .hits)
            blocked_emails = []
            for user in blocked_users:
                username = user.get('username')
                user_email = user.get('email')
                blocked_emails.append(username)
                if user_email and user_email != username:
                    blocked_emails.append(user_email)
            ids_to_delete = _get_contact_ids_to_delete(set(blocked_emails))
            if stdout:
                stdout.write(f"Found {len(ids_to_delete)} id(s) to delete.")
            num_deleted = sum(_delete_hubspot_contact(vid) for vid in ids_to_delete)
            metrics_counter(
                'commcare.hubspot_data.deleted_user.blocked_domain',
                num_deleted,
                tags={
                    'domain': domain,
                    'ids_deleted': ids_to_delete,
                }
            )

def is_hubspot_js_allowed_for_request(request):
    """
    This determines whether a particular request is allowed to load any
    Hubspot javascript, even when Hubspot analytics is supported by the
    environment. This is done to ensure that no accidental Hubspot popups and
    scripts are executed on projects that have explicitly asked to be excluded
    from Hubspot analytics.
    :param request: HttpRequest
    :return: boolean (True if Hubspot javascript is allowed)
    """
    if not settings.IS_SAAS_ENVIRONMENT:
        return False

    is_hubspot_js_allowed = True
    if hasattr(request, 'subscription'):
        try:
            account = request.subscription.account
            is_hubspot_js_allowed = not account.block_hubspot_data_for_all_users
        except BillingAccount.DoesNotExist:
            pass
    return is_hubspot_js_allowed
<commit_msg>instead of logging errors to sentry, keep track of errors in datadog to see if there are any trends<commit_after>import math
import time
import requests
import logging

from django.conf import settings

from corehq.util.metrics import metrics_counter
from corehq.apps.accounting.models import Subscription, BillingAccount
from corehq.apps.es.users import UserES
from corehq.apps.users.models import WebUser, CommCareUser

logger = logging.getLogger('analytics')

MAX_API_RETRIES = 5

ALLOWED_CONVERSIONS = [
    'Blog',
    'Case study/ Ev. Base',
    'Contact us',
    'Event',
    'Live Chat',
    'Newletter',
    'Newsletter',
    'Offer',
    'Outbound',
    'Paid Ads',
    'Video',
]

def get_meta(request):
    return {
        'HTTP_X_FORWARDED_FOR': request.META.get('HTTP_X_FORWARDED_FOR'),
        'REMOTE_ADDR': request.META.get('REMOTE_ADDR'),
    }

def analytics_enabled_for_email(email_address):
    from corehq.apps.users.models import CouchUser
    user = CouchUser.get_by_username(email_address)
    return user.analytics_enabled if user else True

def is_domain_blocked_from_hubspot(domain):
    return Subscription.visible_objects.filter(
        is_active=True,
        subscriber__domain=domain,
        account__is_active=True,
        account__block_hubspot_data_for_all_users=True,
    ).exists()

def hubspot_enabled_for_user(user):
    """
    Check if a user (or account policy) has given permission for Hubspot
    analytics to be synced for a given user.
    :param user: CouchUser or WebUser
    :return: Boolean (True if hubspot is enabled/allowed)
    """
    if isinstance(user, WebUser):
        web_user = user
    else:
        web_user = WebUser.get_by_username(user.username)
    if web_user:
        for domain in web_user.get_domains():
            if is_domain_blocked_from_hubspot(domain):
                return False
    else:
        commcare_user = CommCareUser.get_by_username(user.username)
        if is_domain_blocked_from_hubspot(commcare_user.domain):
            return False
    return user.analytics_enabled

def hubspot_enabled_for_email(email_address):
    from corehq.apps.users.models import CouchUser
    user = CouchUser.get_by_username(email_address)
    return hubspot_enabled_for_user(user) if user else True

def get_blocked_hubspot_domains():
    """
    Get the list of domains / project spaces that have active subscriptions
    with accounts that have blocked hubspot data.
    :return: list
    """
    return list(Subscription.visible_objects.filter(
        account__block_hubspot_data_for_all_users=True,
        is_active=True,
        account__is_active=True,
    ).values_list('subscriber__domain', flat=True))

def get_blocked_hubspot_accounts():
    return [
        f'{account[1]} - ID # {account[0]}'
        for account in BillingAccount.objects.filter(
            block_hubspot_data_for_all_users=True,
            is_active=True,
        ).values_list('id', 'name')
    ]

def get_instance_string():
    instance = settings.ANALYTICS_CONFIG.get('HQ_INSTANCE', '')
    env = '' if instance == 'www' else instance + '_'
    return env

def _delete_hubspot_contact(vid, retry_num=0):
    """
    Permanently deletes a Hubspot contact.
    :param vid:  (the contact ID)
    :param retry_num: the number of the current retry attempt
    :return: boolean if contact was deleted
    """
    if retry_num > 0:
        time.sleep(10)  # wait 10 seconds if this is another retry attempt

    api_key = settings.ANALYTICS_IDS.get('HUBSPOT_API_KEY', None)
    if api_key:
        try:
            req = requests.delete(
                f'https://api.hubapi.com/contacts/v1/contact/vid/{vid}',
                params={
                    'hapikey': api_key,
                }
            )
            if req.status_code == 404:
                return False
            req.raise_for_status()
        except (ConnectionError, requests.exceptions.HTTPError) as e:
            metrics_counter(
                'commcare.hubspot_data.retry.delete_hubspot_contact',
                tags={
                    'vid': vid,
                }
            )
            if retry_num <= MAX_API_RETRIES:
                return _delete_hubspot_contact(vid, retry_num + 1)
            else:
                metrics_counter(
                    'commcare.hubspot_data.error.delete_hubspot_contact',
                    tags={
                        'vid': vid,
                        'error': str(e),
                    }
                )
        else:
            return True
    return False

def _get_contact_ids_to_delete(list_of_emails, retry_num=0):
    """
    Gets a list of Contact IDs on Hubspot from a list of emails.
    If an email in the list doesn't exist on Hubspot, it's simply ignored.
    We also check the list returned from HubSpot to ensure that users
    who engaged with HubSpot under an allowed first conversion action are
    not removed from HubSpot.
    :param list_of_emails:
    :param retry_num: the number of the current retry attempt
    :return: list of contact ids
    """
    if retry_num > 0:
        time.sleep(10)  # wait 10 seconds if this is another retry attempt

    api_key = settings.ANALYTICS_IDS.get('HUBSPOT_API_KEY', None)
    if api_key:
        try:
            req = requests.get(
                "https://api.hubapi.com/contacts/v1/contact/emails/batch/",
                params={
                    'hapikey': api_key,
                    'email': list_of_emails,
                },
            )
            if req.status_code == 404:
                return []
            req.raise_for_status()
        except (ConnectionError, requests.exceptions.HTTPError) as e:
            metrics_counter(
                'commcare.hubspot_data.retry.get_contact_ids_for_emails',
                tags={
                    'emails': list_of_emails.join(', '),
                }
            )
            if retry_num <= MAX_API_RETRIES:
                return _get_contact_ids_to_delete(list_of_emails, retry_num + 1)
            else:
                metrics_counter(
                    'commcare.hubspot_data.error.get_contact_ids_for_emails',
                    tags={
                        'emails': list_of_emails.join(', '),
                        'error': str(e),
                    }
                )
        else:
            ids_to_delete = []
            for contact_id, data in req.json().items():
                first_conversion_status = data.get(
                    'properties', {}
                ).get('first_conversion_clustered_', {}).get('value')

                # If a user's first conversion IS in the allowed list, it means
                # they have directly interacted with us and we want to keep them
                # in the email list. However, we will not send project data
                # about them to HubSpot as they will still be blocked from
                # updates in track_periodic_data.
                if first_conversion_status not in ALLOWED_CONVERSIONS:
                    ids_to_delete.append(contact_id)
            return ids_to_delete
    return []

def remove_blocked_domain_contacts_from_hubspot(stdout=None):
    """
    Removes contacts from Hubspot that are members of blocked domains.
    :param stdout: the stdout of a management command (if applicable)
    """
    blocked_domains = get_blocked_hubspot_domains()
    for domain in blocked_domains:
        if stdout:
            stdout.write(f"\n\nChecking DOMAIN {domain}...")
        user_query = UserES().domain(domain).source(['email', 'username'])

        total_users = user_query.count()
        chunk_size = 30  # Hubspot recommends fewer than 100 emails per request
        num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))

        for chunk in range(num_chunks):
            blocked_users = (user_query
                             .size(chunk_size)
                             .start(chunk * chunk_size)
                             .run()
                             .hits)
            blocked_emails = []
            for user in blocked_users:
                username = user.get('username')
                user_email = user.get('email')
                blocked_emails.append(username)
                if user_email and user_email != username:
                    blocked_emails.append(user_email)
            ids_to_delete = _get_contact_ids_to_delete(set(blocked_emails))
            if stdout:
                stdout.write(f"Found {len(ids_to_delete)} id(s) to delete.")
            num_deleted = sum(_delete_hubspot_contact(vid) for vid in ids_to_delete)
            metrics_counter(
                'commcare.hubspot_data.deleted_user.blocked_domain',
                num_deleted,
                tags={
                    'domain': domain,
                    'ids_deleted': ids_to_delete,
                }
            )

def is_hubspot_js_allowed_for_request(request):
    """
    This determines whether a particular request is allowed to load any
    Hubspot javascript, even when Hubspot analytics is supported by the
    environment. This is done to ensure that no accidental Hubspot popups and
    scripts are executed on projects that have explicitly asked to be excluded
    from Hubspot analytics.
    :param request: HttpRequest
    :return: boolean (True if Hubspot javascript is allowed)
    """
    if not settings.IS_SAAS_ENVIRONMENT:
        return False

    is_hubspot_js_allowed = True
    if hasattr(request, 'subscription'):
        try:
            account = request.subscription.account
            is_hubspot_js_allowed = not account.block_hubspot_data_for_all_users
        except BillingAccount.DoesNotExist:
            pass
    return is_hubspot_js_allowed
<|endoftext|>


<commit_before>import logging

from allauth.socialaccount.models import SocialToken

from django.conf import settings

from .models import GithubProject, GithubOrganization
from tastyapi import apiv2

log = logging.getLogger(__name__)

def make_github_project(user, org, privacy, repo_json):
    if (repo_json['private'] is True and privacy == 'private' or
           repo_json['private'] is False and privacy == 'public'):
        project, created = GithubProject.objects.get_or_create(
            full_name=repo_json['full_name'],
        )
        if project.organization and project.organization != org:
            log.debug('Not importing %s because mismatched orgs' % repo_json['name'])
            return None
        else:
            project.organization = org
        project.users.add(user)
        project.name = repo_json['name']
        project.description = repo_json['description']
        project.git_url = repo_json['git_url']
        project.ssh_url = repo_json['ssh_url']
        project.html_url = repo_json['html_url']
        project.json = repo_json
        project.save()
        return project
    else:
        log.debug('Not importing %s because mismatched type' % repo_json['name'])

def make_github_organization(user, org_json):
    org, created = GithubOrganization.objects.get_or_create(
        login=org_json.get('login'),
    )
    org.html_url = org_json.get('html_url')
    org.name = org_json.get('name')
    org.email = org_json.get('email')
    org.json = org_json
    org.users.add(user)
    return org

def get_token_for_project(project, force_local=False):
    token = None
    try:
        if getattr(settings, 'DONT_HIT_DB', True) and not force_local:
            token = apiv2.project(project.pk).token().get()
        else:
            for user in project.users.all():
                tokens = SocialToken.objects.filter(account__user__username=user.username, app__provider='github')
                if tokens.exists():
                    token = tokens[0].token
    except Exception:
        log.error('Failed to get token for user', exc_info=True)
    return token
<commit_msg>Grab the token from the response<commit_after>import logging

from allauth.socialaccount.models import SocialToken

from django.conf import settings

from .models import GithubProject, GithubOrganization
from tastyapi import apiv2

log = logging.getLogger(__name__)

def make_github_project(user, org, privacy, repo_json):
    if (repo_json['private'] is True and privacy == 'private' or
           repo_json['private'] is False and privacy == 'public'):
        project, created = GithubProject.objects.get_or_create(
            full_name=repo_json['full_name'],
        )
        if project.organization and project.organization != org:
            log.debug('Not importing %s because mismatched orgs' % repo_json['name'])
            return None
        else:
            project.organization = org
        project.users.add(user)
        project.name = repo_json['name']
        project.description = repo_json['description']
        project.git_url = repo_json['git_url']
        project.ssh_url = repo_json['ssh_url']
        project.html_url = repo_json['html_url']
        project.json = repo_json
        project.save()
        return project
    else:
        log.debug('Not importing %s because mismatched type' % repo_json['name'])

def make_github_organization(user, org_json):
    org, created = GithubOrganization.objects.get_or_create(
        login=org_json.get('login'),
    )
    org.html_url = org_json.get('html_url')
    org.name = org_json.get('name')
    org.email = org_json.get('email')
    org.json = org_json
    org.users.add(user)
    return org

def get_token_for_project(project, force_local=False):
    token = None
    try:
        if getattr(settings, 'DONT_HIT_DB', True) and not force_local:
            token = apiv2.project(project.pk).token().get()['token']
        else:
            for user in project.users.all():
                tokens = SocialToken.objects.filter(account__user__username=user.username, app__provider='github')
                if tokens.exists():
                    token = tokens[0].token
    except Exception:
        log.error('Failed to get token for user', exc_info=True)
    return token
<|endoftext|>