<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

from collections import namedtuple
import json
import logging
from settings import MAX_VALUE_OF_PUBLISH_SEQUENCE
from superdesk.celery_app import update_key, set_key
from superdesk import get_resource_service
from eve.utils import ParsedRequest
from superdesk.utils import ListCursor
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.errors import SuperdeskApiError

logger = logging.getLogger(__name__)

subscriber_types = ['broadcast', 'digital', 'wire']
SUBSCRIBER_TYPES = namedtuple('SUBSCRIBER_TYPES', ['BROADCAST', 'DIGITAL', 'WIRE'])(*subscriber_types)

class SubscribersResource(Resource):
    schema = {
        'name': {
            'type': 'string',
            'iunique': True,
            'required': True
        },
        'media_type': {
            'type': 'string'
        },
        'geo_restrictions': {
            'type': 'string',
            'nullable': True
        },
        'subscriber_type': {
            'type': 'string',
            'allowed': subscriber_types,
            'required': True
        },
        'sequence_num_settings': {
            'type': 'dict',
            'schema': {
                'min': {'type': 'integer'},
                'max': {'type': 'integer'}
            },
            'required': True
        },
        'email': {
            'type': 'email',
            'empty': False,
            'required': True
        },
        'is_active': {
            'type': 'boolean',
            'default': True
        },
        'critical_errors': {
            'type': 'dict',
            'keyschema': {
                'type': 'boolean'
            }
        },
        'last_closed': {
            'type': 'dict',
            'schema': {
                'closed_at': {'type': 'datetime'},
                'closed_by': Resource.rel('users', nullable=True),
                'message': {'type': 'string'}
            }
        },
        'destinations': {
            'type': 'list',
            'required': True,
            "minlength": 1,
            'schema': {
                'type': 'dict',
                'schema': {
                    'name': {'type': 'string', 'required': True, 'empty': False},
                    'format': {'type': 'string', 'required': True},
                    'delivery_type': {'type': 'string', 'required': True},
                    'config': {'type': 'dict'}
                }
            }
        },
        'publish_filter': {
            'type': 'dict',
            'schema': {
                'filter_id': Resource.rel('publish_filters', nullable=True),
                'filter_type': {
                    'type': 'string',
                    'allowed': ['blocking', 'permitting'],
                    'default': 'blocking'
                }
            },
            'nullable': True
        },
        'global_filters': {
            'type': 'dict',
            'keyschema': {
                'type': 'boolean'
            }
        }
    }

    datasource = {'default_sort': [('_created', -1)]}

    item_methods = ['GET', 'PATCH', 'PUT']

    privileges = {'POST': 'subscribers', 'PATCH': 'subscribers'}

class SubscribersService(BaseService):
    def get(self, req, lookup):
        if req is None:
            req = ParsedRequest()
        if req.args and req.args.get('filter_condition'):
            filter_condition = json.loads(req.args.get('filter_condition'))
            return ListCursor(self._get_subscribers_by_filter_condition(filter_condition))
        return super().get(req=req, lookup=lookup)

    def on_create(self, docs):
        for doc in docs:
            self._validate_seq_num_settings(doc)

    def on_update(self, updates, original):
        self._validate_seq_num_settings(updates)

    def _get_subscribers_by_filter_condition(self, filter_condition):
        req = ParsedRequest()
        all_subscribers = list(super().get(req=req, lookup=None))
        selected_subscribers = {}

        filter_condition_service = get_resource_service('filter_conditions')
        publish_filter_service = get_resource_service('publish_filters')
        existing_filter_conditions = filter_condition_service._check_similar(filter_condition)
        for fc in existing_filter_conditions:
            existing_publish_filters = publish_filter_service._get_publish_filters_by_filter_condition(fc['_id'])
            for pf in existing_publish_filters:
                if pf.get('is_global', False):
                    for s in all_subscribers:
                        if any(gf for gf in s.get('global_filters', []) if str(s['_id']) in gf):
                            selected_subscribers[s['_id']] = s
                else:
                    for s in all_subscribers:
                        if s.get('publish_filter') and \
                            'filter_id' in s['publish_filter'] and \
                                s['publish_filter']['filter_id'] == pf['_id']:
                            selected_subscribers[s['_id']] = s

        return list(selected_subscribers.values())

    def _validate_seq_num_settings(self, subscriber):
        """
        Validates the 'sequence_num_settings' property if present in subscriber. Below are the validation rules:
            1.  If min value is present then it should be greater than 0
            2.  If min is present and max value isn't available then it's defaulted to MAX_VALUE_OF_PUBLISH_SEQUENCE

        :return: True if validation succeeds otherwise return False.
        """

        if subscriber.get('sequence_num_settings'):
            min = subscriber.get('sequence_num_settings').get('min', 1)
            max = subscriber.get('sequence_num_settings').get('max', MAX_VALUE_OF_PUBLISH_SEQUENCE)

            if min <= 0:
                raise SuperdeskApiError.badRequestError(payload={"sequence_num_settings.min": 1},
                                                        message="Value of Minimum in Sequence Number Settings should "
                                                                "be greater than 0")

            if min >= max:
                raise SuperdeskApiError.badRequestError(payload={"sequence_num_settings.min": 1},
                                                        message="Value of Minimum in Sequence Number Settings should "
                                                                "be less than the value of Maximum")

            del subscriber['sequence_num_settings']
            subscriber['sequence_num_settings'] = {"min": min, "max": max}

        return True

    def generate_sequence_number(self, subscriber):
        """
        Generates Published Sequence Number for the passed subscriber
        """

        assert (subscriber is not None), "Subscriber can't be null"

        sequence_key_name = "{subscriber_name}_subscriber_seq".format(subscriber_name=subscriber.get('name')).lower()
        sequence_number = update_key(sequence_key_name, flag=True)

        max_seq_number = MAX_VALUE_OF_PUBLISH_SEQUENCE

        if subscriber.get('sequence_num_settings'):
            if sequence_number == 0 or sequence_number == 1:
                sequence_number = subscriber['sequence_num_settings']['min']
                set_key(sequence_key_name, value=sequence_number)

            max_seq_number = subscriber['sequence_num_settings']['max']

        if sequence_number == max_seq_number:
            set_key(sequence_key_name)

        return sequence_number
<commit_msg>fix(filter-search) - Return matching global filters<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

from collections import namedtuple
import json
import logging
from settings import MAX_VALUE_OF_PUBLISH_SEQUENCE
from superdesk.celery_app import update_key, set_key
from superdesk import get_resource_service
from eve.utils import ParsedRequest
from superdesk.utils import ListCursor
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.errors import SuperdeskApiError

logger = logging.getLogger(__name__)

subscriber_types = ['broadcast', 'digital', 'wire']
SUBSCRIBER_TYPES = namedtuple('SUBSCRIBER_TYPES', ['BROADCAST', 'DIGITAL', 'WIRE'])(*subscriber_types)

class SubscribersResource(Resource):
    schema = {
        'name': {
            'type': 'string',
            'iunique': True,
            'required': True
        },
        'media_type': {
            'type': 'string'
        },
        'geo_restrictions': {
            'type': 'string',
            'nullable': True
        },
        'subscriber_type': {
            'type': 'string',
            'allowed': subscriber_types,
            'required': True
        },
        'sequence_num_settings': {
            'type': 'dict',
            'schema': {
                'min': {'type': 'integer'},
                'max': {'type': 'integer'}
            },
            'required': True
        },
        'email': {
            'type': 'email',
            'empty': False,
            'required': True
        },
        'is_active': {
            'type': 'boolean',
            'default': True
        },
        'critical_errors': {
            'type': 'dict',
            'keyschema': {
                'type': 'boolean'
            }
        },
        'last_closed': {
            'type': 'dict',
            'schema': {
                'closed_at': {'type': 'datetime'},
                'closed_by': Resource.rel('users', nullable=True),
                'message': {'type': 'string'}
            }
        },
        'destinations': {
            'type': 'list',
            'required': True,
            "minlength": 1,
            'schema': {
                'type': 'dict',
                'schema': {
                    'name': {'type': 'string', 'required': True, 'empty': False},
                    'format': {'type': 'string', 'required': True},
                    'delivery_type': {'type': 'string', 'required': True},
                    'config': {'type': 'dict'}
                }
            }
        },
        'publish_filter': {
            'type': 'dict',
            'schema': {
                'filter_id': Resource.rel('publish_filters', nullable=True),
                'filter_type': {
                    'type': 'string',
                    'allowed': ['blocking', 'permitting'],
                    'default': 'blocking'
                }
            },
            'nullable': True
        },
        'global_filters': {
            'type': 'dict',
            'keyschema': {
                'type': 'boolean'
            }
        }
    }

    datasource = {'default_sort': [('_created', -1)]}

    item_methods = ['GET', 'PATCH', 'PUT']

    privileges = {'POST': 'subscribers', 'PATCH': 'subscribers'}

class SubscribersService(BaseService):
    def get(self, req, lookup):
        if req is None:
            req = ParsedRequest()
        if req.args and req.args.get('filter_condition'):
            filter_condition = json.loads(req.args.get('filter_condition'))
            return ListCursor(self._get_subscribers_by_filter_condition(filter_condition))
        return super().get(req=req, lookup=lookup)

    def on_create(self, docs):
        for doc in docs:
            self._validate_seq_num_settings(doc)

    def on_update(self, updates, original):
        self._validate_seq_num_settings(updates)

    def _get_subscribers_by_filter_condition(self, filter_condition):
        """
        Searches all subscribers that has a publish filter with the given filter condition
        If filter condition is used in a global filter then it returns all
        subscribers that not disabled the global filter.
        :param filter_condition: Filter condition to test
        :return: List of subscribers
        """
        req = ParsedRequest()
        all_subscribers = list(super().get(req=req, lookup=None))
        selected_subscribers = {}

        filter_condition_service = get_resource_service('filter_conditions')
        publish_filter_service = get_resource_service('publish_filters')
        existing_filter_conditions = filter_condition_service._check_similar(filter_condition)
        for fc in existing_filter_conditions:
            existing_publish_filters = publish_filter_service._get_publish_filters_by_filter_condition(fc['_id'])
            for pf in existing_publish_filters:
                if pf.get('is_global', False):
                    for s in all_subscribers:
                        if any(gf for gf in s.get('global_filters', []) if str(s['_id']) not in gf):
                            selected_subscribers[s['_id']] = s
                else:
                    for s in all_subscribers:
                        if s.get('publish_filter') and \
                            'filter_id' in s['publish_filter'] and \
                                s['publish_filter']['filter_id'] == pf['_id']:
                            selected_subscribers[s['_id']] = s

        return list(selected_subscribers.values())

    def _validate_seq_num_settings(self, subscriber):
        """
        Validates the 'sequence_num_settings' property if present in subscriber. Below are the validation rules:
            1.  If min value is present then it should be greater than 0
            2.  If min is present and max value isn't available then it's defaulted to MAX_VALUE_OF_PUBLISH_SEQUENCE

        :return: True if validation succeeds otherwise return False.
        """

        if subscriber.get('sequence_num_settings'):
            min = subscriber.get('sequence_num_settings').get('min', 1)
            max = subscriber.get('sequence_num_settings').get('max', MAX_VALUE_OF_PUBLISH_SEQUENCE)

            if min <= 0:
                raise SuperdeskApiError.badRequestError(payload={"sequence_num_settings.min": 1},
                                                        message="Value of Minimum in Sequence Number Settings should "
                                                                "be greater than 0")

            if min >= max:
                raise SuperdeskApiError.badRequestError(payload={"sequence_num_settings.min": 1},
                                                        message="Value of Minimum in Sequence Number Settings should "
                                                                "be less than the value of Maximum")

            del subscriber['sequence_num_settings']
            subscriber['sequence_num_settings'] = {"min": min, "max": max}

        return True

    def generate_sequence_number(self, subscriber):
        """
        Generates Published Sequence Number for the passed subscriber
        """

        assert (subscriber is not None), "Subscriber can't be null"

        sequence_key_name = "{subscriber_name}_subscriber_seq".format(subscriber_name=subscriber.get('name')).lower()
        sequence_number = update_key(sequence_key_name, flag=True)

        max_seq_number = MAX_VALUE_OF_PUBLISH_SEQUENCE

        if subscriber.get('sequence_num_settings'):
            if sequence_number == 0 or sequence_number == 1:
                sequence_number = subscriber['sequence_num_settings']['min']
                set_key(sequence_key_name, value=sequence_number)

            max_seq_number = subscriber['sequence_num_settings']['max']

        if sequence_number == max_seq_number:
            set_key(sequence_key_name)

        return sequence_number
<|endoftext|>


<commit_before>from django.core import serializers
import simplejson as json
import pytest
from django.core.urlresolvers import reverse
from rest_framework.test import APITestCase
from rest_framework import status

from silver.models import Provider
from silver.tests.factories import ProviderFactory, AdminUserFactory

class TestProviderEndpoints(APITestCase):

    def setUp(self):
        admin_user = AdminUserFactory.create()
        self.client.force_authenticate(user=admin_user)

    def _filter_providers(self, *args, **kwargs):
        return Provider.objects.filter(*args, **kwargs)

    def test_post_valid_provider(self):
        url = reverse('provider-list')
        data = {
            "name": "TestProvider",
            "company": "S.C. Timisoara S.R.L",
            "address_1": "Address",
            "country": "RO",
            "city": "Timisoara",
            "zip_code": "300300",
            "flow": "proforma",
            "invoice_series": "TestSeries",
            "invoice_starting_number": 1,
            "proforma_series": "TestSeries",
            "proforma_starting_number": 1,
        }
        response = self.client.post(url, data)

        assert response.status_code == status.HTTP_201_CREATED
        assert response.data == {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': u'TestProvider',
            'company': u'S.C. Timisoara S.R.L',
            'email': None,
            'address_1': u'Address',
            'address_2': None,
            'city': u'Timisoara',
            'state': None,
            'zip_code': u'300300',
            'country': u'RO',
            'extra': None,
            'flow': 'proforma',
            'invoice_series': 'TestSeries',
            "invoice_starting_number": 1,
            "proforma_series": "TestSeries",
            "proforma_starting_number": 1,
        }
        qs = self._filter_providers()
        assert qs.count() == 1

    def test_post_provider_without_required_fields(self):
        url = reverse('provider-list')
        complete_data = {
            "name": "TestProvider",
            'company': u'S.C. Timisoara S.R.L',
            "address_1": "Address",
            "country": "RO",
            "city": "Timisoara",
            "zip_code": "300300",
            "invoice_series": "TheSeries",
        }
        required_fields = ['company', 'address_1', 'country', 'city',
                           'zip_code', 'flow', 'invoice_series',
                           'invoice_starting_number']

        for field in required_fields:
            temp_data = complete_data.copy()
            try:
                temp_data.pop(field)
            except KeyError:
                pytest.xfail("Required field %s for Provider not provided"
                             " in the test data." % field)

            response = self.client.post(url, temp_data)

            assert response.status_code == 400
            for response_item in response.data.iteritems():
                field_name = response_item[0]
                valid_responses = [
                    (field_name, ['This field may not be blank.']),
                    (field_name, ['This field is required.'])]
                assert response_item in valid_responses

            qs = self._filter_providers()
            assert qs.count() == 0

    def test_get_providers(self):
        batch_size = 40
        ProviderFactory.create_batch(batch_size)
        url = reverse('provider-list')
        response = self.client.get(url)

        full_url = None
        for field in response.data:
            full_url = field.get('url', None)
            if full_url:
                break
        if full_url:
            domain = full_url.split('/')[2]
            full_url = full_url.split(domain)[0] + domain + url

        assert response.status_code == status.HTTP_200_OK
        assert response._headers['x-result-count'] == ('X-Result-Count',
                                                       str(batch_size))
        assert response._headers['link'] == \
            ('Link', '<' + full_url + '?page=2>; rel="next", ' +
             '<' + full_url + '?page=2>; rel="last", ' +
             '<' + full_url + '?page=1>; rel="first"')

        response = self.client.get(url + '?page=2')

        assert response.status_code == status.HTTP_200_OK
        assert response._headers['x-result-count'] == ('X-Result-Count',
                                                       str(batch_size))
        assert response._headers['link'] == \
            ('Link', '<' + full_url + '?page=1>; rel="prev", ' +
             '<' + full_url + '?page=2>; rel="last", ' +
             '<' + full_url + '?page=1>; rel="first"')

    def test_POST_bulk_providers(self):
        providers = ProviderFactory.create_batch(5)

        raw_providers = json.loads(serializers.serialize('json', providers))

        serialized_providers = []
        for item in raw_providers:
            serialized_providers.append(item['fields'])

        url = reverse('provider-list')
        response = self.client.post(url, data=json.dumps(serialized_providers),
                                    content_type='application/json')

        assert response.status_code == status.HTTP_201_CREATED
        assert len(response.data) == 5

    def test_get_provider(self):
        ProviderFactory.reset_sequence(1)
        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})

        response = self.client.get(url)

        assert response.status_code == 200
        assert response.data == {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': 'Name1',
            'company': 'Company1',
            'flow': 'proforma',
            'invoice_series': 'InvoiceSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,
            'email': 'some1@email.com',
            'address_1': 'Address11',
            'address_2': 'Address21',
            'city': 'City1',
            'state': 'State1',
            'zip_code': '1',
            'country': u'AL',
            'extra': 'Extra1'
        }

    def test_get_unexisting_provider(self):
        url = reverse('provider-detail', kwargs={'pk': 1})
        response = self.client.get(url)

        assert response.status_code == status.HTTP_404_NOT_FOUND

    def test_put_provider_correctly(self):
        ProviderFactory.reset_sequence(1)
        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})
        new_data = {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': 'TestProvider',
            'company': 'TheNewCompany',
            'email': 'a@a.com',
            'address_1': 'address',
            'city': 'City',
            'zip_code': '1',
            'country': 'RO',
            'flow': 'proforma',
            'invoice_series': 'NewSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,

        }

        response = self.client.put(url, data=new_data)

        assert response.status_code == status.HTTP_200_OK
        assert response.data == {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': 'TestProvider',
            'company': 'TheNewCompany',
            'flow': 'proforma',
            'email': 'a@a.com',
            'address_1': 'address',
            'address_2': 'Address21',
            'city': 'City',
            'state': 'State1',
            'zip_code': '1',
            'country': 'RO',
            'extra': 'Extra1',
            'flow': 'proforma',
            'invoice_series': 'NewSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,
        }

    def test_put_provider_without_required_field(self):
        """
         .. note::

             The test does not verify each required field, because the test
         test_create_provider_without_required_fields does this and since the
         creation will fail the update will fail too. This is more of a
         sanity test, to check if the correct view is called and if it does
         what's supposed to do for at least one field.
         """

        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})
        new_data = {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'email': 'a@a.com',
            'address_1': 'address',
            'city': 'City',
            'zip_code': '1',
            'country': 'RO',
            'flow': 'invoice',
            'invoice_series': 'NSeries',
            'invoice_starting_number': 2,
        }

        response = self.client.put(url, data=new_data)

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.data == {'name': ['This field is required.']}

    def test_patch_provider(self):
        ProviderFactory.reset_sequence(1)
        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})
        new_data = {
            'company': 'TheNewCompany',  # The changed field
            'address_1': 'Address11',
            'flow': 'proforma',
            'invoice_series': 'InvoiceSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,
            'city': 'City1',
            'zip_code': '1',
            'country': u'AL',
        }

        response = self.client.patch(url, data=new_data)

        assert response.status_code == 200
        assert response.data == {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': 'Name1',
            'company': 'TheNewCompany',
            'flow': 'proforma',
            'invoice_series': 'InvoiceSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,
            'email': 'some1@email.com',
            'address_1': 'Address11',
            'address_2': 'Address21',
            'city': 'City1',
            'state': 'State1',
            'zip_code': '1',
            'country': u'AL',
            'extra': 'Extra1'
        }

    def test_delete_provider(self):
        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})
        response = self.client.delete(url)

        assert response.status_code == status.HTTP_204_NO_CONTENT

    def test_delete_unexisting_provider(self):
        url = reverse('provider-detail', kwargs={'pk': 1})
        response = self.client.delete(url)

        assert response.status_code == status.HTTP_404_NOT_FOUND
<commit_msg>Fixed plan endpoint tests.<commit_after>from django.core import serializers
import simplejson as json
import pytest
from django.core.urlresolvers import reverse
from rest_framework.test import APITestCase
from rest_framework import status

from silver.models import Provider
from silver.tests.factories import ProviderFactory, AdminUserFactory

class TestProviderEndpoints(APITestCase):

    def setUp(self):
        admin_user = AdminUserFactory.create()
        self.client.force_authenticate(user=admin_user)

    def _filter_providers(self, *args, **kwargs):
        return Provider.objects.filter(*args, **kwargs)

    def test_post_valid_provider(self):
        url = reverse('provider-list')
        data = {
            "name": "TestProvider",
            "company": "S.C. Timisoara S.R.L",
            "address_1": "Address",
            "country": "RO",
            "city": "Timisoara",
            "zip_code": "300300",
            "flow": "proforma",
            "invoice_series": "TestSeries",
            "invoice_starting_number": 1,
            "proforma_series": "TestSeries",
            "proforma_starting_number": 1,
        }
        response = self.client.post(url, data)

        assert response.status_code == status.HTTP_201_CREATED
        assert response.data == {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': u'TestProvider',
            'company': u'S.C. Timisoara S.R.L',
            'email': None,
            'address_1': u'Address',
            'address_2': None,
            'city': u'Timisoara',
            'state': None,
            'zip_code': u'300300',
            'country': u'RO',
            'extra': None,
            'flow': 'proforma',
            'invoice_series': 'TestSeries',
            "invoice_starting_number": 1,
            "proforma_series": "TestSeries",
            "proforma_starting_number": 1,
        }
        qs = self._filter_providers()
        assert qs.count() == 1

    def test_post_provider_without_required_fields(self):
        url = reverse('provider-list')
        complete_data = {
            "name": "TestProvider",
            'company': u'S.C. Timisoara S.R.L',
            "address_1": "Address",
            "country": "RO",
            "city": "Timisoara",
            "zip_code": "300300",
            "invoice_series": "TheSeries",
            "invoice_starting_number": 300,
            "flow": "proforma",
            "proforma_series": "TheSecondSeries",
            "proforma_starting_number": 360,
        }
        required_fields = ['address_1', 'country', 'city', 'name',
                           'invoice_series', 'invoice_starting_number',
                           'proforma_series', 'proforma_starting_number']

        for field in required_fields:
            temp_data = complete_data.copy()
            try:
                temp_data.pop(field)
            except KeyError:
                pytest.xfail("Required field %s for Provider not provided"
                             " in the test data." % field)

            response = self.client.post(url, temp_data)

            assert response.status_code == 400

            for response_item in response.data.iteritems():
                field_name = response_item[0]
                if field_name in ['id', 'url']:
                    continue

                valid_responses = [
                    (field_name, ['This field may not be blank.']),
                    (field_name, ['This field is required.']),
                    (field_name, ['This field is required as the chosen flow '
                                  'is proforma.'])]
                assert response_item in valid_responses

            qs = self._filter_providers()
            assert qs.count() == 0

    def test_get_providers(self):
        batch_size = 40
        ProviderFactory.create_batch(batch_size)
        url = reverse('provider-list')
        response = self.client.get(url)

        full_url = None
        for field in response.data:
            full_url = field.get('url', None)
            if full_url:
                break
        if full_url:
            domain = full_url.split('/')[2]
            full_url = full_url.split(domain)[0] + domain + url

        assert response.status_code == status.HTTP_200_OK
        assert response._headers['link'] == \
            ('Link', '<' + full_url + '?page=2; rel="next">, ' +
             '<' + full_url + '?page=1; rel="first">, ' +
             '<' + full_url + '?page=2; rel="last">')

        response = self.client.get(url + '?page=2')

        assert response.status_code == status.HTTP_200_OK
        assert response._headers['link'] == \
            ('Link', '<' + full_url + '; rel="prev">, ' +
             '<' + full_url + '?page=1; rel="first">, ' +
             '<' + full_url + '?page=2; rel="last">')

    def test_POST_bulk_providers(self):
        providers = ProviderFactory.create_batch(5)

        raw_providers = json.loads(serializers.serialize('json', providers))

        serialized_providers = []
        for item in raw_providers:
            serialized_providers.append(item['fields'])

        url = reverse('provider-list')
        response = self.client.post(url, data=json.dumps(serialized_providers),
                                    content_type='application/json')

        assert response.status_code == status.HTTP_201_CREATED
        assert len(response.data) == 5

    def test_get_provider(self):
        ProviderFactory.reset_sequence(1)
        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})

        response = self.client.get(url)

        assert response.status_code == 200
        assert response.data == {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': 'Name1',
            'company': 'Company1',
            'flow': 'proforma',
            'invoice_series': 'InvoiceSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,
            'email': 'some1@email.com',
            'address_1': 'Address11',
            'address_2': 'Address21',
            'city': 'City1',
            'state': 'State1',
            'zip_code': '1',
            'country': u'AL',
            'extra': 'Extra1'
        }

    def test_get_unexisting_provider(self):
        url = reverse('provider-detail', kwargs={'pk': 1})
        response = self.client.get(url)

        assert response.status_code == status.HTTP_404_NOT_FOUND

    def test_put_provider_correctly(self):
        ProviderFactory.reset_sequence(1)
        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})
        new_data = {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': 'TestProvider',
            'company': 'TheNewCompany',
            'email': 'a@a.com',
            'address_1': 'address',
            'city': 'City',
            'zip_code': '1',
            'country': 'RO',
            'flow': 'proforma',
            'invoice_series': 'NewSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,

        }

        response = self.client.put(url, data=new_data)

        assert response.status_code == status.HTTP_200_OK
        assert response.data == {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': 'TestProvider',
            'company': 'TheNewCompany',
            'flow': 'proforma',
            'email': 'a@a.com',
            'address_1': 'address',
            'address_2': 'Address21',
            'city': 'City',
            'state': 'State1',
            'zip_code': '1',
            'country': 'RO',
            'extra': 'Extra1',
            'flow': 'proforma',
            'invoice_series': 'NewSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,
        }

    def test_put_provider_without_required_field(self):
        """
         .. note::

             The test does not verify each required field, because the test
         test_create_provider_without_required_fields does this and since the
         creation will fail the update will fail too. This is more of a
         sanity test, to check if the correct view is called and if it does
         what's supposed to do for at least one field.
         """

        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})
        new_data = {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'email': 'a@a.com',
            'address_1': 'address',
            'city': 'City',
            'zip_code': '1',
            'country': 'RO',
            'flow': 'invoice',
            'invoice_series': 'NSeries',
            'invoice_starting_number': 2,
        }

        response = self.client.put(url, data=new_data)

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.data == {'name': ['This field is required.']}

    def test_patch_provider(self):
        ProviderFactory.reset_sequence(1)
        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})
        new_data = {
            'company': 'TheNewCompany',  # The changed field
            'address_1': 'Address11',
            'flow': 'proforma',
            'invoice_series': 'InvoiceSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,
            'city': 'City1',
            'zip_code': '1',
            'country': u'AL',
        }

        response = self.client.patch(url, data=new_data)

        assert response.status_code == 200
        assert response.data == {
            'id': 1,
            'url': 'http://testserver/providers/1/',
            'name': 'Name1',
            'company': 'TheNewCompany',
            'flow': 'proforma',
            'invoice_series': 'InvoiceSeries',
            'invoice_starting_number': 1,
            'proforma_series': 'ProformaSeries',
            'proforma_starting_number': 1,
            'email': 'some1@email.com',
            'address_1': 'Address11',
            'address_2': 'Address21',
            'city': 'City1',
            'state': 'State1',
            'zip_code': '1',
            'country': u'AL',
            'extra': 'Extra1'
        }

    def test_delete_provider(self):
        ProviderFactory.create()

        url = reverse('provider-detail', kwargs={'pk': 1})
        response = self.client.delete(url)

        assert response.status_code == status.HTTP_204_NO_CONTENT

    def test_delete_unexisting_provider(self):
        url = reverse('provider-detail', kwargs={'pk': 1})
        response = self.client.delete(url)

        assert response.status_code == status.HTTP_404_NOT_FOUND
<|endoftext|>


<commit_before>'''
RMaxAgentClass.py: Class for an RMaxAgent from [Strehl, Li and Littman 2009].

Notes:
    - Assumes WLOG reward function codomain is [0,1] (so RMAX is 1.0)
'''
import math
import random
from collections import defaultdict

import numpy as np

from simple_rl.agents.AgentClass import Agent

class RMaxAgent(Agent):
    '''
    Implementation for an R-Max Agent [Strehl, Li and Littman 2009]
    https://jmlr.org/papers/volume10/strehl09a/strehl09a.pdf
    pseudocode described in the PDF
    '''

    def __init__(self,
                states,
                actions,
                gamma=0.95,
                s_a_threshold=2,
                epsilon_one=0.99,
                max_reward=1.0,
                name="RMax",
                custom_q_init=None):
        """
        Args:
            states: a list of states in the MDP. Simply range(n_states) would suffice
            actions: a list of actions in the MDP. Simply range(n_actions) would suffice
            gamma: discount factor
            s_a_threshold: the number of (s, a) transitions, first seen, to build the model of the MDP
            epsilon_one: param used to decide how many iterations of value iterations to do
            max_reward: the expected maximum reward of the MDP
            name: name of the agent
            custom_q_init: a (n_states, n_actions) numpy array specifying the initial Q values to use
        """
        self.name = name
        self.states = list(states)
        Agent.__init__(self, name=name, actions=actions, gamma=gamma)
        self.rmax = max_reward
        self.s_a_threshold = s_a_threshold
        self.custom_q_init = custom_q_init
        self.epsilon_one = epsilon_one
        self.max_num_value_iter = math.ceil(np.log(1/(self.epsilon_one * (1 - self.gamma))) / (1 - self.gamma))

        self.reset()

    def reset(self):
        '''
        Summary:
            Resets the agent back to its tabular asa config.
        '''
        self.rewards = np.zeros((len(self.states), len(self.actions)))  # rewards are assumed to be state-action based R(s, a)
        self.transitions = np.zeros((len(self.states), len(self.actions), len(self.states)))
        self.s_a_counts = np.zeros((len(self.states), len(self.actions)))
        self.prev_state = None
        self.prev_action = None

        if self.custom_q_init:
            self.q_func = self.custom_q_init
        else:
            self.q_func = np.ones((len(self.states), len(self.actions))) * self.rmax * 1/(1-self.gamma)

    def act(self, state, reward):
        # Compute best action by argmaxing over Q values of all possible s,a pairs
        action = self.get_max_q_action(state)

        # Update given s, a, r, s' : self.prev_state, self.prev_action, reward, state
        self.update(self.prev_state, self.prev_action, reward, state)

        # Update pointers.
        self.prev_action = action
        self.prev_state = state

        return action

    def update(self, state, action, reward, next_state):
        '''
        Args:
            state (State)
            action (str)
            reward (float)
            next_state (State)

        Summary:
            Updates T and R.
        '''
        if state != None and action != None:
            if self.s_a_counts[state][action] < self.s_a_threshold:
                # Add new data points if we haven't seen this s-a enough.
                self.rewards[state][action] += reward
                self.s_a_counts[state][action] += 1
                self.transitions[state][action][next_state] += 1

                if self.s_a_counts[state][action] == self.s_a_threshold:
                    self.value_iteration()

    def value_iteration(self):
        '''
        Do some iterations of value iteration to compute the q values
        Only update the (s, a) pairs that have enough experiences seen
        Q(s, a) = R(s, a) + gamma * \sum_s' T(s, a, s') * max_a' Q(s', a')
        '''
        # mask for update
        mask = self.s_a_counts >= self.s_a_threshold
        pseudo_count = np.where(self.s_a_counts == 0, 1, self.s_a_counts)  # avoid divide by zero

        # build the reward model
        empirical_reward_mat = self.rewards / pseudo_count

        # build the transition model: assume self-loop if there's not enough data
        # assume a self-loop if there's not enough data
        empirical_transition_mat = self.transitions / pseudo_count[:, :, None]
        # only masked positions should be trusted, otherwise self transition

        empirical_transition_mat[~mask] = self._self_transition_mat[~mask]
        assert np.all(empirical_transition_mat.sum(axis=-1) == 1)

        # compute the update for every (s, a), but only apply the ones that needed with a mask
        for i in range(self.max_num_value_iter):
            v = np.max(self.q_func, axis=-1)
            new_q = empirical_reward_mat + self.gamma * np.einsum("san,n->sa", empirical_transition_mat, v)
            if np.all(np.abs(self.q_func[mask] - new_q[mask]) < 1e-4):
                break
            self.q_func[mask] = new_q[mask]

    def _self_transition_mat(self):
        '''
        create a transition matrix where each state just transition to itself
        '''
        self_transition_mat = np.zeros_like(self.transitions)
        self_transition_mat[np.arange(len(self.states)), :, np.arange(len(self.states))] = 1
        return self_transition_mat

    def get_value(self, state):
        '''
        Args:
            state (State)

        Returns:
            (float)
        '''
        return self.get_max_q_value(state)

    def _compute_max_qval_action_pair(self, state):
        '''
        Args:
            state (State)

        Returns:
            (tuple) --> (float, str): where the float is the Qval, str is the action.
        '''
        # Grab random initial action in case all equal
        if np.all(self.q_func[state] == self.q_func[state, 0]):
            best_action = self.rng.choice(self.actions)
        else:
            best_action =np.argmax(self.q_func[state])
        max_q_val = self.q_func[state][best_action]

        return max_q_val, best_action

    def get_max_q_action(self, state):
        '''
        Args:
            state (State)

        Returns:
            (str): The string associated with the action with highest Q value.
        '''
        return self._compute_max_qval_action_pair(state)[1]

    def get_max_q_value(self, state):
        '''
        Args:
            state (State)

        Returns:
            (float): The Q value of the best action in this state.
        '''
        return self._compute_max_qval_action_pair(state)[0]

    def get_q_value(self, state, action):
        '''
        Args:
            state (State)
            action (str)

        Returns:
            (float)
        '''

        return self.q_func[state][action]
<commit_msg>bug fixes for typos in RMAX<commit_after>'''
RMaxAgentClass.py: Class for an RMaxAgent from [Strehl, Li and Littman 2009].

Notes:
    - Assumes WLOG reward function codomain is [0,1] (so RMAX is 1.0)
'''
import math
import random
from collections import defaultdict

import numpy as np

from simple_rl.agents.AgentClass import Agent

class RMaxAgent(Agent):
    '''
    Implementation for an R-Max Agent [Strehl, Li and Littman 2009]
    https://jmlr.org/papers/volume10/strehl09a/strehl09a.pdf
    pseudocode described in the PDF
    '''

    def __init__(self,
                states,
                actions,
                gamma=0.95,
                s_a_threshold=2,
                epsilon_one=0.99,
                max_reward=1.0,
                name="RMax",
                custom_q_init=None):
        """
        Args:
            states: a list of states in the MDP. Simply range(n_states) would suffice
            actions: a list of actions in the MDP. Simply range(n_actions) would suffice
            gamma: discount factor
            s_a_threshold: the number of (s, a) transitions, first seen, to build the model of the MDP
            epsilon_one: param used to decide how many iterations of value iterations to do
            max_reward: the expected maximum reward of the MDP
            name: name of the agent
            custom_q_init: a (n_states, n_actions) numpy array specifying the initial Q values to use
        """
        self.name = name
        self.states = list(states)
        Agent.__init__(self, name=name, actions=actions, gamma=gamma)
        self.rmax = max_reward
        self.s_a_threshold = s_a_threshold
        self.custom_q_init = custom_q_init
        self.epsilon_one = epsilon_one
        self.max_num_value_iter = math.ceil(np.log(1/(self.epsilon_one * (1 - self.gamma))) / (1 - self.gamma))

        self.reset()

    def reset(self):
        '''
        Summary:
            Resets the agent back to its tabular asa config.
        '''
        self.rewards = np.zeros((len(self.states), len(self.actions)))  # rewards are assumed to be state-action based R(s, a)
        self.transitions = np.zeros((len(self.states), len(self.actions), len(self.states)))
        self.s_a_counts = np.zeros((len(self.states), len(self.actions)))
        self.prev_state = None
        self.prev_action = None

        if self.custom_q_init:
            self.q_func = self.custom_q_init
        else:
            self.q_func = np.ones((len(self.states), len(self.actions))) * self.rmax * 1/(1-self.gamma)

    def act(self, state, reward):
        # Compute best action by argmaxing over Q values of all possible s,a pairs
        action = self.get_max_q_action(state)

        # Update given s, a, r, s' : self.prev_state, self.prev_action, reward, state
        self.update(self.prev_state, self.prev_action, reward, state)

        # Update pointers.
        self.prev_action = action
        self.prev_state = state

        return action

    def update(self, state, action, reward, next_state):
        '''
        Args:
            state (State)
            action (str)
            reward (float)
            next_state (State)

        Summary:
            Updates T and R.
        '''
        if state != None and action != None:
            if self.s_a_counts[state][action] < self.s_a_threshold:
                # Add new data points if we haven't seen this s-a enough.
                self.rewards[state][action] += reward
                self.s_a_counts[state][action] += 1
                self.transitions[state][action][next_state] += 1

                if self.s_a_counts[state][action] == self.s_a_threshold:
                    self.value_iteration()

    def value_iteration(self):
        '''
        Do some iterations of value iteration to compute the q values
        Only update the (s, a) pairs that have enough experiences seen
        Q(s, a) = R(s, a) + gamma * \sum_s' T(s, a, s') * max_a' Q(s', a')
        '''
        # mask for update
        mask = self.s_a_counts >= self.s_a_threshold
        pseudo_count = np.where(self.s_a_counts == 0, 1, self.s_a_counts)  # avoid divide by zero

        # build the reward model
        empirical_reward_mat = self.rewards / pseudo_count

        # build the transition model: assume self-loop if there's not enough data
        # assume a self-loop if there's not enough data
        empirical_transition_mat = self.transitions / pseudo_count[:, :, None]
        # only masked positions should be trusted, otherwise self transition

        empirical_transition_mat[~mask] = self._self_transition_mat()[~mask]
        assert np.all(empirical_transition_mat.sum(axis=-1) == 1)

        # compute the update for every (s, a), but only apply the ones that needed with a mask
        for i in range(self.max_num_value_iter):
            v = np.max(self.q_func, axis=-1)
            new_q = empirical_reward_mat + self.gamma * np.einsum("san,n->sa", empirical_transition_mat, v)
            if np.all(np.abs(self.q_func[mask] - new_q[mask]) < 1e-4):
                break
            self.q_func[mask] = new_q[mask]

    def _self_transition_mat(self):
        '''
        create a transition matrix where each state just transition to itself
        '''
        self_transition_mat = np.zeros_like(self.transitions)
        self_transition_mat[np.arange(len(self.states)), :, np.arange(len(self.states))] = 1
        return self_transition_mat

    def get_value(self, state):
        '''
        Args:
            state (State)

        Returns:
            (float)
        '''
        return self.get_max_q_value(state)

    def _compute_max_qval_action_pair(self, state):
        '''
        Args:
            state (State)

        Returns:
            (tuple) --> (float, str): where the float is the Qval, str is the action.
        '''
        # Grab random initial action in case all equal
        if np.all(self.q_func[state] == self.q_func[state, 0]):
            best_action = random.choice(self.actions)
        else:
            best_action =np.argmax(self.q_func[state])
        max_q_val = self.q_func[state][best_action]

        return max_q_val, best_action

    def get_max_q_action(self, state):
        '''
        Args:
            state (State)

        Returns:
            (str): The string associated with the action with highest Q value.
        '''
        return self._compute_max_qval_action_pair(state)[1]

    def get_max_q_value(self, state):
        '''
        Args:
            state (State)

        Returns:
            (float): The Q value of the best action in this state.
        '''
        return self._compute_max_qval_action_pair(state)[0]

    def get_q_value(self, state, action):
        '''
        Args:
            state (State)
            action (str)

        Returns:
            (float)
        '''

        return self.q_func[state][action]
<|endoftext|>


<commit_before>import simuvex

class transmit(simuvex.SimProcedure):
    #pylint:disable=arguments-differ

    def run(self, fd, buf, count, tx_bytes):
        if ABSTRACT_MEMORY in self.state.options:
            data = self.state.mem_expr(buf, count)
            self.state.posix.write(fd, data, count)

            self.state.store_mem(tx_bytes, count, endness='Iend_LE')

        else:
            if self.state.satisfiable(extra_constraints=[count != 0]):
                data = self.state.mem_expr(buf, count)
                self.state.posix.write(fd, data, count)

            self.state.store_mem(tx_bytes, count, endness='Iend_LE', condition=tx_bytes != 0)

        # TODO: transmit failure
        return self.state.se.BVV(0, self.state.arch.bits)

from simuvex.s_options import ABSTRACT_MEMORY
<commit_msg>special handler for fastpath in CGC transmit syscall to avoid memory issue.<commit_after>import simuvex

class transmit(simuvex.SimProcedure):
    #pylint:disable=arguments-differ

    def run(self, fd, buf, count, tx_bytes):

        if self.state.mode == 'fastpath':
            # Special case for CFG generation
            self.state.store_mem(tx_bytes, count, endness='Iend_LE')
            return self.state.se.BVV(0, self.state.arch.bits)

        if ABSTRACT_MEMORY in self.state.options:
            data = self.state.mem_expr(buf, count)
            self.state.posix.write(fd, data, count)

            self.state.store_mem(tx_bytes, count, endness='Iend_LE')

        else:
            if self.state.satisfiable(extra_constraints=[count != 0]):
                data = self.state.mem_expr(buf, count)
                self.state.posix.write(fd, data, count)

            self.state.store_mem(tx_bytes, count, endness='Iend_LE', condition=tx_bytes != 0)

        # TODO: transmit failure
        return self.state.se.BVV(0, self.state.arch.bits)

from simuvex.s_options import ABSTRACT_MEMORY
<|endoftext|>


<commit_before># Copyright (c) Wieland Hoffmann
# License: MIT, see LICENSE for details
from ..schema import SCHEMA
from .types import *
from functools import partial
from logging import getLogger
from sqlalchemy.orm import class_mapper
from sqlalchemy.orm.interfaces import ONETOMANY, MANYTOONE
from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty

logger = getLogger("sir")

def unique_split_paths(paths):
    """
    For each path in ``paths``, yield each of its continuous subpaths.
    If a subpath appears in multiple paths, it will be yielded only once.

    :param [str] paths:
    :rtype: iterator over str
    """
    seen_paths = set()
    for path in paths:
        splits = path.split(".")
        split_length = len(splits)
        for i in xrange(1, split_length + 1):
                join = ".".join(splits[:i])
                if join not in seen_paths:
                    seen_paths.add(join)
                    yield join

def walk_path(model, path):
    """
    Walk ``path`` beginning at ``model`` and return a
    :class:`~sir.trigger_generation.PathPart` object representing a selection
    along that path. Also return the name of the last table seen while
    following ``path``

    :param model: A :ref:`declarative <sqla:declarative_toplevel>` class.
    :param str path:
    """
    current_model = model
    path_length = path.count(".")
    path_part = None
    outermost_path_part = None
    innermost_table_name = None

    for i, path_elem in enumerate(path.split(".")):
        column = getattr(current_model, path_elem)
        prop = column.property

        if isinstance(prop, RelationshipProperty):
            mapper = class_mapper(current_model)
            pk = mapper.primary_key[0].name
            tablename = mapper.mapped_table.name
            innermost_table_name = prop.table.name

            if prop.direction == ONETOMANY:
                new_path_part = OneToManyPathPart(tablename, pk)
                if i == path_length:
                        remote_column = list(prop.remote_side)[0].name
                        inner = ColumnPathPart("", remote_column)
                        new_path_part.inner = inner
            elif prop.direction == MANYTOONE:
                new_path_part = ManyToOnePathPart(tablename, pk,
                                                  column.key)

            current_model = prop.mapper.class_

        elif isinstance(prop, ColumnProperty):
            # We're not interested in columns because the relationship handling
            # takes care of selections on primary keys etc.
            return None, None

        if path_part is None:
            path_part = new_path_part
            outermost_path_part = path_part
        else:
            path_part.inner = new_path_part
            path_part = new_path_part

    if path_part.inner is None:
        # The path ended in a relationship property
        path_part.inner = ColumnPathPart("", "id")

    return outermost_path_part, innermost_table_name

def write_triggers_to_file(triggerfile, functionfile,
                           generators, entityname, table, path, select):
    """
    Write deletion, insertion and update triggers to ``triggerfile``.

    :param file triggerfile:
    :param file functionfile:
    :param generators:
    :type generators: [TriggerGenerator]
    :param str entityname:
    :param str table:
    :param str path:
    :param str select:
    """
    for generator in generators:
        gen = generator(entityname, table, path, select)
        functionfile.write(gen.function)
        triggerfile.write(gen.trigger)

def write_direct_triggers(triggerfile, functionfile, entityname, model):
    """
    :param file triggerfile:
    :param str entityname:
    :param model: A :ref:`declarative <sqla:declarative_toplevel>` class.
    """
    id_select = "SELECT id FROM {table} WHERE {pk} = {{new_or_old}}.{pk}"
    mapper = class_mapper(model)
    pk = mapper.primary_key[0].name
    tablename = mapper.mapped_table.name

    write_triggers_to_file(triggerfile,
                           functionfile,
                           generators=(GIDDeleteTriggerGenerator,
                                       InsertTriggerGenerator,
                                       UpdateTriggerGenerator),
                           entityname=entityname,
                           table=tablename,
                           path="direct",
                           select=id_select.format(pk=pk, table=tablename))

def write_header(file_):
    file_.write("-- Automatically generated, do not edit\n")
    file_.write("\set ON_ERROR_STOP 1\n")
    file_.write("BEGIN;\n")

def write_footer(file_):
    file_.write("COMMIT;\n")

def generate_triggers(args):
    """
    The entry point for this module. This function gets called from
    :func:`~sir.__main__.main`.
    """
    trigger_filename = args["trigger_file"]
    function_filename = args["function_file"]
    with open(trigger_filename, "w") as triggerfile,\
         open(function_filename, "w") as functionfile:
        write_header(triggerfile)
        write_header(functionfile)
        for entityname, e in SCHEMA.iteritems():
            writer = partial(write_triggers_to_file,
                             generators=(DeleteTriggerGenerator,
                                         InsertTriggerGenerator,
                                         UpdateTriggerGenerator),
                             triggerfile=triggerfile,
                             functionfile=functionfile,
                             entityname=entityname)
            paths = unique_split_paths([path for field in e.fields for path in
                                        field.paths])

            write_direct_triggers(triggerfile, functionfile, entityname, e.model)

            for path in paths:
                select, table = walk_path(e.model, path)
                if select is not None:
                    select = select.render()
                    writer(table=table, path=path, select=select)
        write_footer(triggerfile)
        write_footer(functionfile)
<commit_msg>Missed some docs<commit_after># Copyright (c) Wieland Hoffmann
# License: MIT, see LICENSE for details
from ..schema import SCHEMA
from .types import *
from functools import partial
from logging import getLogger
from sqlalchemy.orm import class_mapper
from sqlalchemy.orm.interfaces import ONETOMANY, MANYTOONE
from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty

logger = getLogger("sir")

def unique_split_paths(paths):
    """
    For each path in ``paths``, yield each of its continuous subpaths.
    If a subpath appears in multiple paths, it will be yielded only once.

    :param [str] paths:
    :rtype: iterator over str
    """
    seen_paths = set()
    for path in paths:
        splits = path.split(".")
        split_length = len(splits)
        for i in xrange(1, split_length + 1):
                join = ".".join(splits[:i])
                if join not in seen_paths:
                    seen_paths.add(join)
                    yield join

def walk_path(model, path):
    """
    Walk ``path`` beginning at ``model`` and return a
    :class:`~sir.trigger_generation.PathPart` object representing a selection
    along that path. Also return the name of the last table seen while
    following ``path``

    :param model: A :ref:`declarative <sqla:declarative_toplevel>` class.
    :param str path:
    """
    current_model = model
    path_length = path.count(".")
    path_part = None
    outermost_path_part = None
    innermost_table_name = None

    for i, path_elem in enumerate(path.split(".")):
        column = getattr(current_model, path_elem)
        prop = column.property

        if isinstance(prop, RelationshipProperty):
            mapper = class_mapper(current_model)
            pk = mapper.primary_key[0].name
            tablename = mapper.mapped_table.name
            innermost_table_name = prop.table.name

            if prop.direction == ONETOMANY:
                new_path_part = OneToManyPathPart(tablename, pk)
                if i == path_length:
                        remote_column = list(prop.remote_side)[0].name
                        inner = ColumnPathPart("", remote_column)
                        new_path_part.inner = inner
            elif prop.direction == MANYTOONE:
                new_path_part = ManyToOnePathPart(tablename, pk,
                                                  column.key)

            current_model = prop.mapper.class_

        elif isinstance(prop, ColumnProperty):
            # We're not interested in columns because the relationship handling
            # takes care of selections on primary keys etc.
            return None, None

        if path_part is None:
            path_part = new_path_part
            outermost_path_part = path_part
        else:
            path_part.inner = new_path_part
            path_part = new_path_part

    if path_part.inner is None:
        # The path ended in a relationship property
        path_part.inner = ColumnPathPart("", "id")

    return outermost_path_part, innermost_table_name

def write_triggers_to_file(triggerfile, functionfile,
                           generators, entityname, table, path, select):
    """
    Write deletion, insertion and update triggers to ``triggerfile``.

    :param file triggerfile:
    :param file functionfile:
    :param generators:
    :type generators: [TriggerGenerator]
    :param str entityname:
    :param str table:
    :param str path:
    :param str select:
    """
    for generator in generators:
        gen = generator(entityname, table, path, select)
        functionfile.write(gen.function)
        triggerfile.write(gen.trigger)

def write_direct_triggers(triggerfile, functionfile, entityname, model):
    """
    :param file triggerfile:
    :param file functionfile:
    :param str entityname:
    :param model: A :ref:`declarative <sqla:declarative_toplevel>` class.
    """
    id_select = "SELECT id FROM {table} WHERE {pk} = {{new_or_old}}.{pk}"
    mapper = class_mapper(model)
    pk = mapper.primary_key[0].name
    tablename = mapper.mapped_table.name

    write_triggers_to_file(triggerfile,
                           functionfile,
                           generators=(GIDDeleteTriggerGenerator,
                                       InsertTriggerGenerator,
                                       UpdateTriggerGenerator),
                           entityname=entityname,
                           table=tablename,
                           path="direct",
                           select=id_select.format(pk=pk, table=tablename))

def write_header(file_):
    """
    :param file file_:
    """
    file_.write("-- Automatically generated, do not edit\n")
    file_.write("\set ON_ERROR_STOP 1\n")
    file_.write("BEGIN;\n")

def write_footer(file_):
    """
    :param file file_:
    """
    file_.write("COMMIT;\n")

def generate_triggers(args):
    """
    The entry point for this module. This function gets called from
    :func:`~sir.__main__.main`.
    """
    trigger_filename = args["trigger_file"]
    function_filename = args["function_file"]
    with open(trigger_filename, "w") as triggerfile,\
         open(function_filename, "w") as functionfile:
        write_header(triggerfile)
        write_header(functionfile)
        for entityname, e in SCHEMA.iteritems():
            writer = partial(write_triggers_to_file,
                             generators=(DeleteTriggerGenerator,
                                         InsertTriggerGenerator,
                                         UpdateTriggerGenerator),
                             triggerfile=triggerfile,
                             functionfile=functionfile,
                             entityname=entityname)
            paths = unique_split_paths([path for field in e.fields for path in
                                        field.paths])

            write_direct_triggers(triggerfile, functionfile, entityname, e.model)

            for path in paths:
                select, table = walk_path(e.model, path)
                if select is not None:
                    select = select.render()
                    writer(table=table, path=path, select=select)
        write_footer(triggerfile)
        write_footer(functionfile)
<|endoftext|>


<commit_before>import pip
import importlib
import math
from typing import Dict
import time

from stp_core.common.log import getlogger

logger = getlogger()

notifierPluginTriggerEvents = {
    'nodeRequestSpike': 'NodeRequestSuspiciousSpike',
    'clusterThroughputSpike': 'ClusterThroughputSuspiciousSpike',
    # TODO: Implement clusterLatencyTooHigh event triggering
    'clusterLatencyTooHigh': 'ClusterLatencyTooHigh',
    'nodeUpgradeScheduled': 'NodeUpgradeScheduled',
    'nodeUpgradeComplete': 'NodeUpgradeComplete',
    'nodeUpgradeFail': 'NodeUpgradeFail',
    'poolUpgradeCancel': 'PoolUpgradeCancel'
}

class PluginManager:
    prefix = 'indynotifier'
    __instance = None

    def __new__(cls):
        if PluginManager.__instance is None:
            PluginManager.__instance = object.__new__(cls)
        return PluginManager.__instance

    def __init__(self):
        self.plugins = []
        self.topics = notifierPluginTriggerEvents
        self.importPlugins()

    def sendMessageUponNodeUpgradeScheduled(
            self, message='Node uprgade has been scheduled'):
        return self._sendMessage(self.topics['nodeUpgradeScheduled'], message)

    def sendMessageUponNodeUpgradeComplete(
            self, message='Node has successfully upgraded.'):
        return self._sendMessage(self.topics['nodeUpgradeComplete'], message)

    def sendMessageUponNodeUpgradeFail(
            self, message='Node upgrade has failed. Please take action.'):
        return self._sendMessage(self.topics['nodeUpgradeFail'], message)

    def sendMessageUponPoolUpgradeCancel(
            self, message='Pool upgrade has been cancelled. Please take action.'):
        return self._sendMessage(self.topics['poolUpgradeCancel'], message)

    def sendMessageUponSuspiciousSpike(
            self,
            event: str,
            historicalData: Dict,
            newVal: float,
            config: Dict,
            nodeName: str,
            enabled: bool):
        assert 'value' in historicalData
        assert 'cnt' in historicalData

        assert 'min_cnt' in config
        assert 'borders_coeff' in config
        assert 'min_activity_threshold' in config
        assert 'use_weighted_borders_coeff' in config
        assert 'enabled' in config

        if not (enabled and config['enabled']):
            logger.trace('Suspicious Spike check is disabled')
            return None

        min_cnt = config['min_cnt']
        val_thres = config['min_activity_threshold']
        borders_coeff = config['borders_coeff']
        use_weighted_borders_coeff = config['use_weighted_borders_coeff']

        val = historicalData['value']
        alpha = 2 / (min_cnt + 1)
        historicalData['value'] = val * (1 - alpha) + newVal * alpha
        historicalData['cnt'] += 1
        cnt = historicalData['cnt']

        if cnt <= min_cnt:
            logger.debug('Not enough data to detect a {} spike'.format(event))
            return None

        if val < val_thres:
            logger.debug('Current activity {} is below threshold level {}'.format(val, val_thres))
            return None

        log_base = 10
        if use_weighted_borders_coeff and cnt > log_base:
            # Weighted coefficient allows to adapt borders in accordance to values,
            # growing values leads to lower borders.
            borders_coeff /= math.log(cnt, log_base)

        lower_bound = val / borders_coeff
        higher_bound = val * borders_coeff
        if lower_bound <= newVal <= higher_bound:
            logger.debug(
                '{}: Actual value {} is within bounds [{}, {}]. Expected value: {}'.format(
                    event, newVal, lower_bound, higher_bound, val))
            return None

        message = '{} suspicious spike has been noticed on node {} at {}. ' \
                  'Actual: {}. Expected: {}. Bounds: [{}, {}].'\
            .format(event, nodeName, time.time(), newVal, val, lower_bound, higher_bound)
        logger.debug(message)
        return self._sendMessage(event, message)

    def importPlugins(self):
        plugins = self._findPlugins()
        self.plugins = []
        i = 0
        for plugin in plugins:
            try:
                module = importlib.import_module(plugin)
                self.plugins.append(module)
                i += 1
                logger.info(
                    "Successfully imported Notifier Plugin: {}".format(plugin))
            except Exception as e:
                logger.error(
                    'Importing Notifier Plugin {} failed due to {}'.format(
                        plugin, e))
        return i, len(plugins)

    def _sendMessage(self, topic, message):
        i = 0
        for plugin in self.plugins:
            try:
                plugin.send_message(topic, message)
                i += 1
            except Exception as e:
                logger.error('Sending message failed for plugin {} due to {}'
                             .format(plugin.__name__, e))
        return i, len(self.plugins)

    def _findPlugins(self):
        return [pkg.key
                for pkg in pip.utils.get_installed_distributions()
                if pkg.key.startswith(PluginManager.prefix)]
<commit_msg>Log found notifier plugins.<commit_after>import pip
import importlib
import math
from typing import Dict
import time

from stp_core.common.log import getlogger

logger = getlogger()

notifierPluginTriggerEvents = {
    'nodeRequestSpike': 'NodeRequestSuspiciousSpike',
    'clusterThroughputSpike': 'ClusterThroughputSuspiciousSpike',
    # TODO: Implement clusterLatencyTooHigh event triggering
    'clusterLatencyTooHigh': 'ClusterLatencyTooHigh',
    'nodeUpgradeScheduled': 'NodeUpgradeScheduled',
    'nodeUpgradeComplete': 'NodeUpgradeComplete',
    'nodeUpgradeFail': 'NodeUpgradeFail',
    'poolUpgradeCancel': 'PoolUpgradeCancel'
}

class PluginManager:
    prefix = 'indynotifier'
    __instance = None

    def __new__(cls):
        if PluginManager.__instance is None:
            PluginManager.__instance = object.__new__(cls)
        return PluginManager.__instance

    def __init__(self):
        self.plugins = []
        self.topics = notifierPluginTriggerEvents
        self.importPlugins()

    def sendMessageUponNodeUpgradeScheduled(
            self, message='Node uprgade has been scheduled'):
        return self._sendMessage(self.topics['nodeUpgradeScheduled'], message)

    def sendMessageUponNodeUpgradeComplete(
            self, message='Node has successfully upgraded.'):
        return self._sendMessage(self.topics['nodeUpgradeComplete'], message)

    def sendMessageUponNodeUpgradeFail(
            self, message='Node upgrade has failed. Please take action.'):
        return self._sendMessage(self.topics['nodeUpgradeFail'], message)

    def sendMessageUponPoolUpgradeCancel(
            self, message='Pool upgrade has been cancelled. Please take action.'):
        return self._sendMessage(self.topics['poolUpgradeCancel'], message)

    def sendMessageUponSuspiciousSpike(
            self,
            event: str,
            historicalData: Dict,
            newVal: float,
            config: Dict,
            nodeName: str,
            enabled: bool):
        assert 'value' in historicalData
        assert 'cnt' in historicalData

        assert 'min_cnt' in config
        assert 'borders_coeff' in config
        assert 'min_activity_threshold' in config
        assert 'use_weighted_borders_coeff' in config
        assert 'enabled' in config

        if not (enabled and config['enabled']):
            logger.trace('Suspicious Spike check is disabled')
            return None

        min_cnt = config['min_cnt']
        val_thres = config['min_activity_threshold']
        borders_coeff = config['borders_coeff']
        use_weighted_borders_coeff = config['use_weighted_borders_coeff']

        val = historicalData['value']
        alpha = 2 / (min_cnt + 1)
        historicalData['value'] = val * (1 - alpha) + newVal * alpha
        historicalData['cnt'] += 1
        cnt = historicalData['cnt']

        if cnt <= min_cnt:
            logger.debug('Not enough data to detect a {} spike'.format(event))
            return None

        if val < val_thres:
            logger.debug('Current activity {} is below threshold level {}'.format(val, val_thres))
            return None

        log_base = 10
        if use_weighted_borders_coeff and cnt > log_base:
            # Weighted coefficient allows to adapt borders in accordance to values,
            # growing values leads to lower borders.
            borders_coeff /= math.log(cnt, log_base)

        lower_bound = val / borders_coeff
        higher_bound = val * borders_coeff
        if lower_bound <= newVal <= higher_bound:
            logger.debug(
                '{}: Actual value {} is within bounds [{}, {}]. Expected value: {}'.format(
                    event, newVal, lower_bound, higher_bound, val))
            return None

        message = '{} suspicious spike has been noticed on node {} at {}. ' \
                  'Actual: {}. Expected: {}. Bounds: [{}, {}].'\
            .format(event, nodeName, time.time(), newVal, val, lower_bound, higher_bound)
        logger.debug(message)
        return self._sendMessage(event, message)

    def importPlugins(self):
        plugins = self._findPlugins()
        logger.info("Found notifier plugins: {}".format(plugins))
        self.plugins = []
        i = 0
        for plugin in plugins:
            try:
                module = importlib.import_module(plugin)
                self.plugins.append(module)
                i += 1
                logger.info(
                    "Successfully imported Notifier Plugin: {}".format(plugin))
            except Exception as e:
                logger.error(
                    'Importing Notifier Plugin {} failed due to {}'.format(
                        plugin, e))
        return i, len(plugins)

    def _sendMessage(self, topic, message):
        i = 0
        for plugin in self.plugins:
            try:
                plugin.send_message(topic, message)
                i += 1
            except Exception as e:
                logger.error('Sending message failed for plugin {} due to {}'
                             .format(plugin.__name__, e))
        return i, len(self.plugins)

    def _findPlugins(self):
        return [pkg.key
                for pkg in pip.utils.get_installed_distributions()
                if pkg.key.startswith(PluginManager.prefix)]
<|endoftext|>


<commit_before>import os
import numpy as np
import cPickle as pk
import time
import threading

from hrl_lib.msg import NumpyArray

## Returns a string that can be used as a timestamp (hours and minutes) in logfiles
# @return timestamp-string
def getTime():
    return '['+time.strftime("%H:%M:%S", time.localtime())+']'

def standard_rad(t):
    if t > 0:
        return ((t + np.pi) % (np.pi * 2))  - np.pi
    else:
        return ((t - np.pi) % (np.pi * -2)) + np.pi

##
# Converts a list of numpy matrices to one large matrix
# @param list_mat the list of little matrices
# @param axis axis to concatenate little matrices
# @return one large numpy matrix
def list_mat_to_mat(list_mat, axis=0):
    return np.concatenate(tuple(list_mat), axis=axis)

## returns current time as a string: year|month|date_hours|min|sec.
## @return current time as a string: year|month|date_hours|min|sec.
def formatted_time():
    date_name = time.strftime('%Y%h%d_%H%M%S', time.localtime())
#    curtime = time.localtime()
#    date_name = time.strftime('%Y%m%d_%I%M%S', curtime)
    return date_name

## read a pickle and return the object.
# @param filename - name of the pkl
# @return - object that had been pickled.
def load_pickle(filename):
    try:
        p = open(filename, 'r')
    except IOError:
        print "hrl_lib.util: Pickle file cannot be opened."
        return None
    picklelicious = pk.load(p)
    p.close()
    return picklelicious

## Pickle an object.
# @param object - object to be pickled
# @param filename - name of the pkl file
def save_pickle(object, filename):
    pickle_file = open(filename, 'w')
    pk.dump(object, pickle_file)
    pickle_file.close()

## Calculate L2 norm for column vectors in a matrix
# @param mat - numpy matrix
def norm(mat):
    return np.power(np.sum(np.power(mat,2), axis=0), 0.5)

def approx_equal(a, b, epsilon=.001):
    return (b < (a+epsilon)) and ((a-epsilon) < b)

def unipolar_limit( x, upper ):
    """ limit the value of x such that
        0 <= x <= upper
    """

    if x > upper:
        x=upper
    if x < 0:
        x=0

    return x

def cart_of_pol(p):
   """ Finds cartesian coordinates of polar points [r, t]' """
   r = p[0,:]
   t = p[1,:]
   x = numpy.multiply(numpy.cos(t), r)
   y = numpy.multiply(numpy.sin(t), r)
   return numpy.vstack((x,y))

def pol_of_cart(p):
    """ Find polar coordinates of cartesian points [x, y]' """
    norm = numpy.linalg.norm(p)
    ang = math.atan2(p[1,0], p[0,0])
    return numpy.matrix([norm, ang]).T

##
# Bound the value of a number to be above lower, and lower than upper
# @return a number
def bound(value, lower, upper):
    if lower >= upper:
        t = lower
        lower = upper
        upper = t
#    print 'bound', value, 'lower', lower, 'upper', upper
    #return min(max(value, lower), upper)
    ret_val = min(max(value, lower), upper)
#    if ret_val != value:
#        print 'ut.boud bounded something.'
    return ret_val

## wraps a numpy array into hrl's datatype for sending np arrays
# over ros.
# @param np array
# @return NumpyArray object (hrl_lib/msg/NumpyArray.msg)
def wrap_np_array(nparr):
    shp = nparr.shape
    npstr = nparr.tostring()
    npdtype = str(nparr.dtype)
    nparr_ros = NumpyArray(None,npstr,shp,npdtype)
    return nparr_ros

## convert hrl's ros wrapped numpy array to a numpy array
# @param NumpyArray object (hrl_lib/msg/NumpyArray.msg)
# @return np array
def unwrap_np_array(nparr_ros):
    npstr,shp,npdtype = nparr_ros.data,nparr_ros.shape,nparr_ros.dtype
    nparr = np.fromstring(npstr,dtype=npdtype)
    nparr = nparr.reshape(shp)
    return nparr

## cartesian product of list of lists.
# code copied from: http://automatthias.wordpress.com/2007/04/28/cartesian-product-of-multiple-sets/
# @return generator. can loop over it, or list(generator) will give
# the entire list.
# NOTE - itertools in python 2.6 provides this functionality. We
# should switch over to it soon.
def cartesian_product(lists, previous_elements = []):
    if len(lists) == 1:
        for elem in lists[0]:
            yield previous_elements + [elem, ]
    else:
        for elem in lists[0]:
            for x in cartesian_product(lists[1:], previous_elements + [elem, ]):
                yield x

## choose n elements from list without replacement.
# adapted code from cartesian_product
# @return generator.
def choose_without_replacement(list, n):
    lists = [list for i in range(n)]
    return _choose_without_replacement(lists)

def _choose_without_replacement(lists,previous_elements=[],ignore_count=0):
    if len(lists) == 1:
        for elem in lists[0][ignore_count:]:
            yield previous_elements + [elem, ]
    else:
        for i,elem in enumerate(lists[0][ignore_count:]):
            for x in _choose_without_replacement(lists[1:],previous_elements + [elem, ],ignore_count+i+1):
                yield x

##
# use festival text to speech to make a soud.
# @param text - string to be said.
def say(text):
    os.system( 'echo "' + text + '" | festival --tts' )

## compute rank of a matrix.
# code copied from:
# http://mail.scipy.org/pipermail/numpy-discussion/2008-February/031218.html
def matrixrank(A,tol=1e-8):
    s = np.linalg.svd(A,compute_uv=0)
    return sum( np.where( s>tol, 1, 0 ) )

<commit_msg>M    hrl_lib/util.py cart_of_pol had three space indent instead of four.<commit_after>import os
import numpy as np
import cPickle as pk
import time
import threading

from hrl_lib.msg import NumpyArray

## Returns a string that can be used as a timestamp (hours and minutes) in logfiles
# @return timestamp-string
def getTime():
    return '['+time.strftime("%H:%M:%S", time.localtime())+']'

def standard_rad(t):
    if t > 0:
        return ((t + np.pi) % (np.pi * 2))  - np.pi
    else:
        return ((t - np.pi) % (np.pi * -2)) + np.pi

##
# Converts a list of numpy matrices to one large matrix
# @param list_mat the list of little matrices
# @param axis axis to concatenate little matrices
# @return one large numpy matrix
def list_mat_to_mat(list_mat, axis=0):
    return np.concatenate(tuple(list_mat), axis=axis)

## returns current time as a string: year|month|date_hours|min|sec.
## @return current time as a string: year|month|date_hours|min|sec.
def formatted_time():
    date_name = time.strftime('%Y%h%d_%H%M%S', time.localtime())
#    curtime = time.localtime()
#    date_name = time.strftime('%Y%m%d_%I%M%S', curtime)
    return date_name

## read a pickle and return the object.
# @param filename - name of the pkl
# @return - object that had been pickled.
def load_pickle(filename):
    try:
        p = open(filename, 'r')
    except IOError:
        print "hrl_lib.util: Pickle file cannot be opened."
        return None
    picklelicious = pk.load(p)
    p.close()
    return picklelicious

## Pickle an object.
# @param object - object to be pickled
# @param filename - name of the pkl file
def save_pickle(object, filename):
    pickle_file = open(filename, 'w')
    pk.dump(object, pickle_file)
    pickle_file.close()

## Calculate L2 norm for column vectors in a matrix
# @param mat - numpy matrix
def norm(mat):
    return np.power(np.sum(np.power(mat,2), axis=0), 0.5)

def approx_equal(a, b, epsilon=.001):
    return (b < (a+epsilon)) and ((a-epsilon) < b)

def unipolar_limit( x, upper ):
    """ limit the value of x such that
        0 <= x <= upper
    """

    if x > upper:
        x=upper
    if x < 0:
        x=0

    return x

def cart_of_pol(p):
    """ Finds cartesian coordinates of polar points [r, t]' """
    r = p[0,:]
    t = p[1,:]
    x = numpy.multiply(numpy.cos(t), r)
    y = numpy.multiply(numpy.sin(t), r)
    return numpy.vstack((x,y))

def pol_of_cart(p):
    """ Find polar coordinates of cartesian points [x, y]' """
    norm = numpy.linalg.norm(p)
    ang = math.atan2(p[1,0], p[0,0])
    return numpy.matrix([norm, ang]).T

##
# Bound the value of a number to be above lower, and lower than upper
# @return a number
def bound(value, lower, upper):
    if lower >= upper:
        t = lower
        lower = upper
        upper = t
#    print 'bound', value, 'lower', lower, 'upper', upper
    #return min(max(value, lower), upper)
    ret_val = min(max(value, lower), upper)
#    if ret_val != value:
#        print 'ut.boud bounded something.'
    return ret_val

## wraps a numpy array into hrl's datatype for sending np arrays
# over ros.
# @param np array
# @return NumpyArray object (hrl_lib/msg/NumpyArray.msg)
def wrap_np_array(nparr):
    shp = nparr.shape
    npstr = nparr.tostring()
    npdtype = str(nparr.dtype)
    nparr_ros = NumpyArray(None,npstr,shp,npdtype)
    return nparr_ros

## convert hrl's ros wrapped numpy array to a numpy array
# @param NumpyArray object (hrl_lib/msg/NumpyArray.msg)
# @return np array
def unwrap_np_array(nparr_ros):
    npstr,shp,npdtype = nparr_ros.data,nparr_ros.shape,nparr_ros.dtype
    nparr = np.fromstring(npstr,dtype=npdtype)
    nparr = nparr.reshape(shp)
    return nparr

## cartesian product of list of lists.
# code copied from: http://automatthias.wordpress.com/2007/04/28/cartesian-product-of-multiple-sets/
# @return generator. can loop over it, or list(generator) will give
# the entire list.
# NOTE - itertools in python 2.6 provides this functionality. We
# should switch over to it soon.
def cartesian_product(lists, previous_elements = []):
    if len(lists) == 1:
        for elem in lists[0]:
            yield previous_elements + [elem, ]
    else:
        for elem in lists[0]:
            for x in cartesian_product(lists[1:], previous_elements + [elem, ]):
                yield x

## choose n elements from list without replacement.
# adapted code from cartesian_product
# @return generator.
def choose_without_replacement(list, n):
    lists = [list for i in range(n)]
    return _choose_without_replacement(lists)

def _choose_without_replacement(lists,previous_elements=[],ignore_count=0):
    if len(lists) == 1:
        for elem in lists[0][ignore_count:]:
            yield previous_elements + [elem, ]
    else:
        for i,elem in enumerate(lists[0][ignore_count:]):
            for x in _choose_without_replacement(lists[1:],previous_elements + [elem, ],ignore_count+i+1):
                yield x

##
# use festival text to speech to make a soud.
# @param text - string to be said.
def say(text):
    os.system( 'echo "' + text + '" | festival --tts' )

## compute rank of a matrix.
# code copied from:
# http://mail.scipy.org/pipermail/numpy-discussion/2008-February/031218.html
def matrixrank(A,tol=1e-8):
    s = np.linalg.svd(A,compute_uv=0)
    return sum( np.where( s>tol, 1, 0 ) )

<|endoftext|>