<commit_before>import ckan.lib.helpers as h
import ckan.model as model
import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import os

from ckanext.romania_theme.logic.auth.delete import (
    romania_theme_package_delete, romania_theme_resource_delete)

def get_number_of_files():
    return model.Session.execute("select count(*) from resource where state = 'active'").first()[0]

def get_number_of_external_links():
    return model.Session.execute("select count(*) from resource where state = 'active' and url not LIKE '%data.gov.ro%'").first()[0]

class Romania_ThemePlugin(plugins.SingletonPlugin):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.ITemplateHelpers)
    plugins.implements(plugins.IResourceController, inherit=True)
    plugins.implements(plugins.IAuthFunctions)

    # IConfigurer
    def update_config(self, config):
        toolkit.add_template_directory(config, 'templates')
        toolkit.add_public_directory(config, 'public')
        toolkit.add_resource('fanstatic', 'romania_theme')

    def get_helpers(self):
        return {'get_number_of_files': get_number_of_files,
                'get_number_of_external_links': get_number_of_external_links}

    # IResourceController
    def before_create(self, context, resource):
        if ('upload' in resource) and (type(resource['upload']) is not unicode) and (resource['upload'].type in ['application/pdf', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/msword']):
            raise toolkit.ValidationError(['Fisierele de tip PDF, DOC sau DOCX nu sunt permise.'])

    # IAuthFunctions
    def get_auth_functions(self):
        return {
            'package_delete': romania_theme_package_delete,
            'resource_delete': romania_theme_resource_delete,
        }
<commit_msg>Solve filename bug when uploading a resource #8<commit_after>import ckan.lib.helpers as h
import ckan.model as model
import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import os

from ckanext.romania_theme.logic.auth.delete import (
    romania_theme_package_delete, romania_theme_resource_delete)

def get_number_of_files():
    return model.Session.execute("select count(*) from resource where state = 'active'").first()[0]

def get_number_of_external_links():
    return model.Session.execute("select count(*) from resource where state = 'active' and url not LIKE '%data.gov.ro%'").first()[0]

class Romania_ThemePlugin(plugins.SingletonPlugin):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.ITemplateHelpers)
    plugins.implements(plugins.IResourceController, inherit=True)
    plugins.implements(plugins.IAuthFunctions)

    # IConfigurer
    def update_config(self, config):
        toolkit.add_template_directory(config, 'templates')
        toolkit.add_public_directory(config, 'public')
        toolkit.add_resource('fanstatic', 'romania_theme')

    def get_helpers(self):
        return {'get_number_of_files': get_number_of_files,
                'get_number_of_external_links': get_number_of_external_links}

    # IResourceController
    def before_create(self, context, resource):
        if ('upload' in resource) and (type(resource['upload']) is not unicode) and (resource['upload'].type in ['application/pdf', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/msword']):
            # If we did not do this, the URL field would contain the filename
            # and people can press finalize afterwards.
            resource['url'] = ''

            raise toolkit.ValidationError(['Fisierele de tip PDF, DOC sau DOCX nu sunt permise.'])

    # IAuthFunctions
    def get_auth_functions(self):
        return {
            'package_delete': romania_theme_package_delete,
            'resource_delete': romania_theme_resource_delete,
        }
<|endoftext|>


<commit_before>import re
import logging
from itertools import zip_longest

from .. import data
from ..reddit import RedditStreamingProcessBase, Reply

USERNAME = 'TagTrain'
USERNAME_FULL = f'u/{USERNAME}'
MEMBER_LIMIT = 3

R_INTRO = f'(?:/?u/{USERNAME} )?'
C_GROUP = '(?:/?u/)?(?P<group>[^ ]+)'
C_MEMBER = '(?:/?u/)?(?P<member>[^ ]+)'
C_NAME = '(?:/?u/)?(?P<name>[^ ]+)'

R_GROUPS = ('groups', re.compile(f'^{R_INTRO}groups$'))
R_HELP = ('help', re.compile(f'^{R_INTRO}(hello|help)$'))
R_USE = ('use', re.compile(f'^{R_INTRO}use {C_GROUP}$'))
R_GROUP = ('group', re.compile(f'^{R_INTRO}group {C_GROUP}$'))
R_CLEAR = ('clear', re.compile(f'^{R_INTRO}clear {C_GROUP}$'))
R_ADD = ('add', re.compile(f'^{R_INTRO}add {C_MEMBER} to {C_GROUP}$'))
R_REMOVE = ('remove', re.compile(f'^{R_INTRO}remove {C_MEMBER} from {C_GROUP}$'))
R_RENAME = ('rename', re.compile(f'^{R_INTRO}rename {C_GROUP} to {C_NAME}$'))

R_ALL = [R_HELP, R_GROUPS, R_GROUP, R_ADD, R_REMOVE, R_USE, R_CLEAR, R_RENAME]

_logger = logging.getLogger('tagtrain')

def _grouper(iterable, n, fillvalue=None):
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)

class TagTrain(RedditStreamingProcessBase):
    def __init__(self, db_path):
        data.init(db_path)

    def _find_command(self, message):
        _logger.debug('_find_command...')
        for n, r in R_ALL:
            for line in message.body.split('\n'):
                rv = r.search(line)
                if rv:
                    return n, rv
        return None, None

    def _reply_help(self, reply, message, match):
        _logger.debug('_reply_help...')
        reply.append(f"""
TagTrain Bot, here to help you mass mention users in a Thread

- Groups are specific to a reddit user.  You can not add/remove/use another reddit user's list.
- Group names need to be unique per reddit user.
- Group names can not contain spaces.  Use dash, underscore, period, etc as a word separator.
- Members can be specified as `name`, `u/name`, or `/u/name`.
- A reddit user can (currently) have unlimited Groups.
- A Group can (currently) have unlimited Members.
- Why not MentionTrain or MessageTrain? TagTrain is nicely alliterative.
- Questions/Comments? Message `/u/c17r`.

In a Comment or Message:

- `u/TagTrain help` or `u/TagTrain hello` - Displays this message.
- `u/TagTrain groups` - Displays name of all your Groups with count of Members.
- `u/TagTrain group <group-name>` - Displays all Members for specified Group.
- `u/TagTrain add <user> to <group-name>` - Adds specified reddit user to the specified Group.
- `u/TagTrain remove <user> to <group-name>` - Removes specified Member from the specified Group, if Group is now empty it is deleted.
- `u/TagTrain` clear <group-name> - Deletes specified Group and Members.
- `u/TagTrain` rename <group-name> to <new-name> - Renames specified Group.

In a Comment:

- `u/TagTrain use <group-name>` - Loops through Members of specified Group, mentioning {MEMBER_LIMIT} members at a time.
""")  # noqa: E501

    def _reply_use(self, reply, message, match):
        _logger.debug('_reply_use...')
        owner_name = message.author.name
        group_name = match.group('group')

        try:
            group = data.find_group(owner_name, group_name)

            reply.append(f'Using Group `{group.name}` to notify {group.member_count} Members')
            for groupings in _grouper(group.members.iterator(), MEMBER_LIMIT):
                tmp = ', '.join([f'u/{member.reddit_name}' for member in groupings if member])
                reply.new_child(f'`{group.reddit_name}` used `TagTrain` Bot to mention you: {tmp}')

        except data.Group.DoesNotExist:
            reply.append(f'Group `{group_name}` does not exist, no Members.')

    def _reply_groups(self, reply, message, match):
        _logger.debug('_reply_groups...')
        owner_name = message.author.name

        groups = data.find_groups(owner_name)

        response = [f'`{g.name}` | {g.member_count}' for g in groups]
        if response:
            response.insert(0, 'Groups | Member Counts')
            response.insert(1, '------ | -------------')
            reply.append('\n'.join(response))
        else:
            reply.append('You have no Groups')

    def _reply_group(self, reply, message, match):
        _logger.debug('_reply_group...')
        owner_name = message.author.name
        group_name = match.group('group')

        try:
            group = data.find_group(owner_name, group_name)
            sorted_members = sorted(group.members, key=lambda x: x.reddit_name.lower())

            response = [
                f'Group `{group.name}` has {group.member_count} Members:',
                '\n'
                ' | | | ',
                ' - | - | - '
            ]
            for groupings in _grouper(sorted_members, MEMBER_LIMIT):
                response.append(' | '.join([f'{"`" + g.reddit_name + "`" if g else ""}' for g in groupings]))
            reply.append('\n'.join(response))

        except data.Group.DoesNotExist:
            reply.append(f'Group `{group_name}` does not exist, no Members.')

    def _reply_clear(self, reply, message, match):
        _logger.debug('_reply_clear')
        owner_name = message.author.name
        group_name = match.group('group')

        try:
            data.remove_group(owner_name, group_name)
            reply.append(f'Group `{group_name}` and all Members removed.')

        except data.Group.DoesNotExist:
            reply.append(f'Group`{group_name}` does not exist.')

    def _reply_add(self, reply, message, match):
        _logger.debug('_reply_add...')
        owner_name = message.author.name
        group_name = match.group('group')
        member_name = match.group('member')

        group, created = data.add_user_to_group(owner_name, group_name, member_name)
        if group.member_count == 1:
            reply.append(f'Group `{group.name}` created with `{member_name}` as the first Member.')
        else:
            if created:
                reply.append(f'`{member_name}` added to Group `{group.name}`, {group.member_count} total Members')
            else:
                reply.append(
                    f'`{member_name}` already Member of Group `{group.name}`, {group.member_count} total Members')

    def _reply_remove(self, reply, message, match):
        _logger.debug('_reply_remove...')
        owner_name = message.author.name
        group_name = match.group('group')
        member_name = match.group('member')

        try:
            group = data.remove_user_from_group(owner_name, group_name, member_name)
        except data.Group.DoesNotExist:
            reply.append(f'Group `{group_name}` does not exist, doing nothing')
            return
        except data.Member.DoesNotExist:
            reply.append(f'`{member_name}` not Member of Group `{group_name}`')
            return

        tmp = f'`{member_name}` removed from Group `{group.name}`, '
        if group.member_count > 0:
            tmp += f'{group.member_count} total Members'
        else:
            tmp += f'Group has no Members left so it was removed'
        reply.append(tmp)

    def _reply_rename(self, reply, message, match):
        _logger.debug('_reply_rename')
        owner_name = message.author.name
        group_name = match.group('group')
        new_name = match.group('name')

        try:
            data.rename_group(owner_name, group_name, new_name)
        except data.Group.DoesNotExist:
            reply.append(f'Group `{group_name}` does not exist, doing nothing')
            return

        reply.append(f'Group `{group_name}` renamed to `{new_name}`')

    def _reply_error(self, reply, message, match):
        _logger.debug('_reply_error...')
        reply.append('Sorry, unknown command. Showing help')
        self._reply_help(reply, message, match)

    def process(self, message):
        _logger.debug('process...')
        name, match = self._find_command(message)
        func = getattr(self, f'_reply_{name}', None)
        reply = Reply()
        if func:
            reply.append('>' + match.group(0))
            reply.append('\n')
            func(reply, message, match)
        else:
            self._reply_error(reply, message, None)
        _logger.debug('Reply is ' + str(reply))
        return reply
<commit_msg>fix typo<commit_after>import re
import logging
from itertools import zip_longest

from .. import data
from ..reddit import RedditStreamingProcessBase, Reply

USERNAME = 'TagTrain'
USERNAME_FULL = f'u/{USERNAME}'
MEMBER_LIMIT = 3

R_INTRO = f'(?:/?u/{USERNAME} )?'
C_GROUP = '(?:/?u/)?(?P<group>[^ ]+)'
C_MEMBER = '(?:/?u/)?(?P<member>[^ ]+)'
C_NAME = '(?:/?u/)?(?P<name>[^ ]+)'

R_GROUPS = ('groups', re.compile(f'^{R_INTRO}groups$'))
R_HELP = ('help', re.compile(f'^{R_INTRO}(hello|help)$'))
R_USE = ('use', re.compile(f'^{R_INTRO}use {C_GROUP}$'))
R_GROUP = ('group', re.compile(f'^{R_INTRO}group {C_GROUP}$'))
R_CLEAR = ('clear', re.compile(f'^{R_INTRO}clear {C_GROUP}$'))
R_ADD = ('add', re.compile(f'^{R_INTRO}add {C_MEMBER} to {C_GROUP}$'))
R_REMOVE = ('remove', re.compile(f'^{R_INTRO}remove {C_MEMBER} from {C_GROUP}$'))
R_RENAME = ('rename', re.compile(f'^{R_INTRO}rename {C_GROUP} to {C_NAME}$'))

R_ALL = [R_HELP, R_GROUPS, R_GROUP, R_ADD, R_REMOVE, R_USE, R_CLEAR, R_RENAME]

_logger = logging.getLogger('tagtrain')

def _grouper(iterable, n, fillvalue=None):
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)

class TagTrain(RedditStreamingProcessBase):
    def __init__(self, db_path):
        data.init(db_path)

    def _find_command(self, message):
        _logger.debug('_find_command...')
        for n, r in R_ALL:
            for line in message.body.split('\n'):
                rv = r.search(line)
                if rv:
                    return n, rv
        return None, None

    def _reply_help(self, reply, message, match):
        _logger.debug('_reply_help...')
        reply.append(f"""
TagTrain Bot, here to help you mass mention users in a Thread

- Groups are specific to a reddit user.  You can not add/remove/use another reddit user's list.
- Group names need to be unique per reddit user.
- Group names can not contain spaces.  Use dash, underscore, period, etc as a word separator.
- Members can be specified as `name`, `u/name`, or `/u/name`.
- A reddit user can (currently) have unlimited Groups.
- A Group can (currently) have unlimited Members.
- Why not MentionTrain or MessageTrain? TagTrain is nicely alliterative.
- Questions/Comments? Message `/u/c17r`.

In a Comment or Message:

- `u/TagTrain help` or `u/TagTrain hello` - Displays this message.
- `u/TagTrain groups` - Displays name of all your Groups with count of Members.
- `u/TagTrain group <group-name>` - Displays all Members for specified Group.
- `u/TagTrain add <user> to <group-name>` - Adds specified reddit user to the specified Group.
- `u/TagTrain remove <user> to <group-name>` - Removes specified Member from the specified Group, if Group is now empty it is deleted.
- `u/TagTrain` clear <group-name> - Deletes specified Group and Members.
- `u/TagTrain` rename <group-name> to <new-name> - Renames specified Group.

In a Comment:

- `u/TagTrain use <group-name>` - Loops through Members of specified Group, mentioning {MEMBER_LIMIT} Members at a time.
""")  # noqa: E501

    def _reply_use(self, reply, message, match):
        _logger.debug('_reply_use...')
        owner_name = message.author.name
        group_name = match.group('group')

        try:
            group = data.find_group(owner_name, group_name)

            reply.append(f'Using Group `{group.name}` to notify {group.member_count} Members')
            for groupings in _grouper(group.members.iterator(), MEMBER_LIMIT):
                tmp = ', '.join([f'u/{member.reddit_name}' for member in groupings if member])
                reply.new_child(f'`{group.reddit_name}` used `TagTrain` Bot to mention you: {tmp}')

        except data.Group.DoesNotExist:
            reply.append(f'Group `{group_name}` does not exist, no Members.')

    def _reply_groups(self, reply, message, match):
        _logger.debug('_reply_groups...')
        owner_name = message.author.name

        groups = data.find_groups(owner_name)

        response = [f'`{g.name}` | {g.member_count}' for g in groups]
        if response:
            response.insert(0, 'Groups | Member Counts')
            response.insert(1, '------ | -------------')
            reply.append('\n'.join(response))
        else:
            reply.append('You have no Groups')

    def _reply_group(self, reply, message, match):
        _logger.debug('_reply_group...')
        owner_name = message.author.name
        group_name = match.group('group')

        try:
            group = data.find_group(owner_name, group_name)
            sorted_members = sorted(group.members, key=lambda x: x.reddit_name.lower())

            response = [
                f'Group `{group.name}` has {group.member_count} Members:',
                '\n'
                ' | | | ',
                ' - | - | - '
            ]
            for groupings in _grouper(sorted_members, MEMBER_LIMIT):
                response.append(' | '.join([f'{"`" + g.reddit_name + "`" if g else ""}' for g in groupings]))
            reply.append('\n'.join(response))

        except data.Group.DoesNotExist:
            reply.append(f'Group `{group_name}` does not exist, no Members.')

    def _reply_clear(self, reply, message, match):
        _logger.debug('_reply_clear')
        owner_name = message.author.name
        group_name = match.group('group')

        try:
            data.remove_group(owner_name, group_name)
            reply.append(f'Group `{group_name}` and all Members removed.')

        except data.Group.DoesNotExist:
            reply.append(f'Group`{group_name}` does not exist.')

    def _reply_add(self, reply, message, match):
        _logger.debug('_reply_add...')
        owner_name = message.author.name
        group_name = match.group('group')
        member_name = match.group('member')

        group, created = data.add_user_to_group(owner_name, group_name, member_name)
        if group.member_count == 1:
            reply.append(f'Group `{group.name}` created with `{member_name}` as the first Member.')
        else:
            if created:
                reply.append(f'`{member_name}` added to Group `{group.name}`, {group.member_count} total Members')
            else:
                reply.append(
                    f'`{member_name}` already Member of Group `{group.name}`, {group.member_count} total Members')

    def _reply_remove(self, reply, message, match):
        _logger.debug('_reply_remove...')
        owner_name = message.author.name
        group_name = match.group('group')
        member_name = match.group('member')

        try:
            group = data.remove_user_from_group(owner_name, group_name, member_name)
        except data.Group.DoesNotExist:
            reply.append(f'Group `{group_name}` does not exist, doing nothing')
            return
        except data.Member.DoesNotExist:
            reply.append(f'`{member_name}` not Member of Group `{group_name}`')
            return

        tmp = f'`{member_name}` removed from Group `{group.name}`, '
        if group.member_count > 0:
            tmp += f'{group.member_count} total Members'
        else:
            tmp += f'Group has no Members left so it was removed'
        reply.append(tmp)

    def _reply_rename(self, reply, message, match):
        _logger.debug('_reply_rename')
        owner_name = message.author.name
        group_name = match.group('group')
        new_name = match.group('name')

        try:
            data.rename_group(owner_name, group_name, new_name)
        except data.Group.DoesNotExist:
            reply.append(f'Group `{group_name}` does not exist, doing nothing')
            return

        reply.append(f'Group `{group_name}` renamed to `{new_name}`')

    def _reply_error(self, reply, message, match):
        _logger.debug('_reply_error...')
        reply.append('Sorry, unknown command. Showing help')
        self._reply_help(reply, message, match)

    def process(self, message):
        _logger.debug('process...')
        name, match = self._find_command(message)
        func = getattr(self, f'_reply_{name}', None)
        reply = Reply()
        if func:
            reply.append('>' + match.group(0))
            reply.append('\n')
            func(reply, message, match)
        else:
            self._reply_error(reply, message, None)
        _logger.debug('Reply is ' + str(reply))
        return reply
<|endoftext|>


<commit_before>'''Arsenal client version'''
#
#  Copyright 2015 CityGrid Media, LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
__version__ = '7.3'
<commit_msg>INFRA-16585: version bump<commit_after>'''Arsenal client version'''
#
#  Copyright 2015 CityGrid Media, LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
__version__ = '7.4'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Tests for BLEU translation evaluation metric
"""

import functools
import io
import unittest

from nltk.data import find
from nltk.translate.bleu_score import modified_precision, brevity_penalty, closest_ref_length
from nltk.translate.bleu_score import sentence_bleu, corpus_bleu, SmoothingFunction

class TestBLEU(unittest.TestCase):
    def test_modified_precision(self):
        """
        Examples from the original BLEU paper
        http://www.aclweb.org/anthology/P02-1040.pdf
        """
        # Example 1: the "the*" example.
        # Reference sentences.
        ref1 = 'the cat is on the mat'.split()
        ref2 = 'there is a cat on the mat'.split()
        # Hypothesis sentence(s).
        hyp1 = 'the the the the the the the'.split()

        references = [ref1, ref2]

        # Testing modified unigram precision.
        hyp1_unigram_precision =  float(modified_precision(references, hyp1, n=1))
        assert (round(hyp1_unigram_precision, 4) == 0.2857)
        # With assertAlmostEqual at 4 place precision.
        self.assertAlmostEqual(hyp1_unigram_precision, 0.28571428, places=4)

        # Testing modified bigram precision.
        assert(float(modified_precision(references, hyp1, n=2)) == 0.0)

        # Example 2: the "of the" example.
        # Reference sentences
        ref1 = str('It is a guide to action that ensures that the military '
                   'will forever heed Party commands').split()
        ref2 = str('It is the guiding principle which guarantees the military '
                   'forces always being under the command of the Party').split()
        ref3 = str('It is the practical guide for the army always to heed '
                   'the directions of the party').split()
        # Hypothesis sentence(s).
        hyp1 = 'of the'.split()

        references = [ref1, ref2, ref3]
        # Testing modified unigram precision.
        assert (float(modified_precision(references, hyp1, n=1)) == 1.0)

        # Testing modified bigram precision.
        assert(float(modified_precision(references, hyp1, n=2)) == 1.0)

        # Example 3: Proper MT outputs.
        hyp1 = str('It is a guide to action which ensures that the military '
                   'always obeys the commands of the party').split()
        hyp2 = str('It is to insure the troops forever hearing the activity '
                   'guidebook that party direct').split()

        references = [ref1, ref2, ref3]

        # Unigram precision.
        hyp1_unigram_precision = float(modified_precision(references, hyp1, n=1))
        hyp2_unigram_precision = float(modified_precision(references, hyp2, n=1))
        # Test unigram precision with assertAlmostEqual at 4 place precision.
        self.assertAlmostEqual(hyp1_unigram_precision, 0.94444444, places=4)
        self.assertAlmostEqual(hyp2_unigram_precision, 0.57142857, places=4)
        # Test unigram precision with rounding.
        assert (round(hyp1_unigram_precision, 4) == 0.9444)
        assert (round(hyp2_unigram_precision, 4) == 0.5714)

        # Bigram precision
        hyp1_bigram_precision = float(modified_precision(references, hyp1, n=2))
        hyp2_bigram_precision = float(modified_precision(references, hyp2, n=2))
        # Test bigram precision with assertAlmostEqual at 4 place precision.
        self.assertAlmostEqual(hyp1_bigram_precision, 0.58823529, places=4)
        self.assertAlmostEqual(hyp2_bigram_precision, 0.07692307, places=4)
        # Test bigram precision with rounding.
        assert (round(hyp1_bigram_precision, 4) == 0.5882)
        assert (round(hyp2_bigram_precision, 4) == 0.0769)

    def test_brevity_penalty(self):
        # Test case from brevity_penalty_closest function in mteval-v13a.pl.
        # Same test cases as in the doctest in nltk.translate.bleu_score.py
        references = [['a'] * 11, ['a'] * 8]
        hypothesis = ['a'] * 7
        hyp_len = len(hypothesis)
        closest_ref_len =  closest_ref_length(references, hyp_len)
        self.assertAlmostEqual(brevity_penalty(closest_ref_len, hyp_len), 0.8669, places=4)

        references = [['a'] * 11, ['a'] * 8, ['a'] * 6, ['a'] * 7]
        hypothesis = ['a'] * 7
        hyp_len = len(hypothesis)
        closest_ref_len =  closest_ref_length(references, hyp_len)
        assert brevity_penalty(closest_ref_len, hyp_len) == 1.0

    def test_zero_matches(self):
        # Test case where there's 0 matches
        references = ['The candidate has no alignment to any of the references'.split()]
        hypothesis = 'John loves Mary'.split()

        # Test BLEU to nth order of n-grams, where n is len(hypothesis).
        for n in range(1,len(hypothesis)):
            weights = [1.0/n] * n # Uniform weights.
            assert(sentence_bleu(references, hypothesis, weights) == 0)

    def test_full_matches(self):
        # Test case where there's 100% matches
        references = ['John loves Mary'.split()]
        hypothesis = 'John loves Mary'.split()

        # Test BLEU to nth order of n-grams, where n is len(hypothesis).
        for n in range(1,len(hypothesis)):
            weights = [1.0/n] * n # Uniform weights.
            assert(sentence_bleu(references, hypothesis, weights) == 1.0)

    def test_partial_matches_hypothesis_longer_than_reference(self):
        references = ['John loves Mary'.split()]
        hypothesis = 'John loves Mary who loves Mike'.split()
        # Since no 4-grams matches were found the result should be zero
        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.0, places=4)
        # Checks that the warning has been raised because len(reference) < 4.
        try:
            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
        except AttributeError:
            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.

#@unittest.skip("Skipping fringe cases for BLEU.")
class TestBLEUFringeCases(unittest.TestCase):

    def test_case_where_n_is_bigger_than_hypothesis_length(self):
        # Test BLEU to nth order of n-grams, where n > len(hypothesis).
        references = ['John loves Mary ?'.split()]
        hypothesis = 'John loves Mary'.split()
        n = len(hypothesis) + 1 #
        weights = [1.0/n] * n # Uniform weights.
        # Since no n-grams matches were found the result should be zero
        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
        self.assertAlmostEqual(sentence_bleu(references, hypothesis, weights), 0.0, places=4)
        # Checks that the warning has been raised because len(hypothesis) < 4.
        try:
            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
        except AttributeError:
            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.

        # Test case where n > len(hypothesis) but so is n > len(reference), and
        # it's a special case where reference == hypothesis.
        references = ['John loves Mary'.split()]
        hypothesis = 'John loves Mary'.split()
        # Since no 4-grams matches were found the result should be zero
        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
        self.assertAlmostEqual(sentence_bleu(references, hypothesis, weights), 0.0, places=4)

    def test_empty_hypothesis(self):
        # Test case where there's hypothesis is empty.
        references = ['The candidate has no alignment to any of the references'.split()]
        hypothesis = []
        assert(sentence_bleu(references, hypothesis) == 0)

    def test_empty_references(self):
        # Test case where there's reference is empty.
        references = [[]]
        hypothesis = 'John loves Mary'.split()
        assert(sentence_bleu(references, hypothesis) == 0)

    def test_empty_references_and_hypothesis(self):
        # Test case where both references and hypothesis is empty.
        references = [[]]
        hypothesis = []
        assert(sentence_bleu(references, hypothesis) == 0)

    def test_reference_or_hypothesis_shorter_than_fourgrams(self):
        # Tese case where the length of reference or hypothesis
        # is shorter than 4.
        references = ['let it go'.split()]
        hypothesis = 'let go it'.split()
        # Checks that the value the hypothesis and reference returns is 0.0
        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.0, places=4)
        # Checks that the warning has been raised.
        try:
            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
        except AttributeError:
            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.

class TestBLEUvsMteval13a(unittest.TestCase):

    def test_corpus_bleu(self):
        ref_file = find('models/wmt15_eval/ref.ru')
        hyp_file = find('models/wmt15_eval/google.ru')
        mteval_output_file = find('models/wmt15_eval/mteval-13a.output')

        # Reads the BLEU scores from the `mteval-13a.output` file.
        # The order of the list corresponds to the order of the ngrams.
        with open(mteval_output_file, 'r') as mteval_fin:
            # The numbers are located in the last 2nd line of the file.
            # The first and 2nd item in the list are the score and system names.
            mteval_bleu_scores = map(float, mteval_fin.readlines()[-2].split()[1:-1])

        with io.open(ref_file, 'r', encoding='utf8') as ref_fin:
            with io.open(hyp_file, 'r', encoding='utf8') as hyp_fin:
                # Whitespace tokenize the file.
                # Note: split() automatically strip().
                hypothesis = list(map(lambda x: x.split(), hyp_fin))
                # Note that the corpus_bleu input is list of list of references.
                references = list(map(lambda x: [x.split()], ref_fin))
                # Without smoothing.
                for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
                    nltk_bleu = corpus_bleu(references, hypothesis, weights=(1.0/i,)*i)
                    # Check that the BLEU scores difference is less than 0.005 .
                    # Note: This is an approximate comparison; as much as
                    #       +/- 0.01 BLEU might be "statistically significant",
                    #       the actual translation quality might not be.
                    assert abs(mteval_bleu - nltk_bleu) < 0.005

                # With the same smoothing method used in mteval-v13a.pl
                chencherry = SmoothingFunction()
                for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
                    nltk_bleu = corpus_bleu(references, hypothesis,
                                            weights=(1.0/i,)*i,
                                            smoothing_function=chencherry.method3)
                    assert abs(mteval_bleu - nltk_bleu) < 0.005

class TestEmulateMultiBLEU(unittest.TestCase):
    def test_corpus_bleu_with_bad_sentence(self):
        hyp = "Teo S yb , oe uNb , R , T t , , t Tue Ar saln S , , 5istsi l , 5oe R ulO sae oR R"
        ref = str("Their tasks include changing a pump on the faulty stokehold ."
                  "Likewise , two species that are very similar in morphology "
                  "were distinguished using genetics .")
        references = [[ref.split()]]
        hypothese = [hyp.split()]
        try: # Check that the warning is raised since no. of 2-grams < 0.
            with self.assertWarns(UserWarning):
                # Verify that the BLEU output is undesired since no. of 2-grams < 0.
                self.assertAlmostEqual(corpus_bleu(references, hypothese), 0.4309, places=4)
        except AttributeError:
            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
        desired_output = corpus_bleu(references, hypothese)
        #assert
        assert desired_output == 0.0
<commit_msg>Update test_bleu.py<commit_after># -*- coding: utf-8 -*-
"""
Tests for BLEU translation evaluation metric
"""

import functools
import io
import unittest

from nltk.data import find
from nltk.translate.bleu_score import modified_precision, brevity_penalty, closest_ref_length
from nltk.translate.bleu_score import sentence_bleu, corpus_bleu, SmoothingFunction

class TestBLEU(unittest.TestCase):
    def test_modified_precision(self):
        """
        Examples from the original BLEU paper
        http://www.aclweb.org/anthology/P02-1040.pdf
        """
        # Example 1: the "the*" example.
        # Reference sentences.
        ref1 = 'the cat is on the mat'.split()
        ref2 = 'there is a cat on the mat'.split()
        # Hypothesis sentence(s).
        hyp1 = 'the the the the the the the'.split()

        references = [ref1, ref2]

        # Testing modified unigram precision.
        hyp1_unigram_precision =  float(modified_precision(references, hyp1, n=1))
        assert (round(hyp1_unigram_precision, 4) == 0.2857)
        # With assertAlmostEqual at 4 place precision.
        self.assertAlmostEqual(hyp1_unigram_precision, 0.28571428, places=4)

        # Testing modified bigram precision.
        assert(float(modified_precision(references, hyp1, n=2)) == 0.0)

        # Example 2: the "of the" example.
        # Reference sentences
        ref1 = str('It is a guide to action that ensures that the military '
                   'will forever heed Party commands').split()
        ref2 = str('It is the guiding principle which guarantees the military '
                   'forces always being under the command of the Party').split()
        ref3 = str('It is the practical guide for the army always to heed '
                   'the directions of the party').split()
        # Hypothesis sentence(s).
        hyp1 = 'of the'.split()

        references = [ref1, ref2, ref3]
        # Testing modified unigram precision.
        assert (float(modified_precision(references, hyp1, n=1)) == 1.0)

        # Testing modified bigram precision.
        assert(float(modified_precision(references, hyp1, n=2)) == 1.0)

        # Example 3: Proper MT outputs.
        hyp1 = str('It is a guide to action which ensures that the military '
                   'always obeys the commands of the party').split()
        hyp2 = str('It is to insure the troops forever hearing the activity '
                   'guidebook that party direct').split()

        references = [ref1, ref2, ref3]

        # Unigram precision.
        hyp1_unigram_precision = float(modified_precision(references, hyp1, n=1))
        hyp2_unigram_precision = float(modified_precision(references, hyp2, n=1))
        # Test unigram precision with assertAlmostEqual at 4 place precision.
        self.assertAlmostEqual(hyp1_unigram_precision, 0.94444444, places=4)
        self.assertAlmostEqual(hyp2_unigram_precision, 0.57142857, places=4)
        # Test unigram precision with rounding.
        assert (round(hyp1_unigram_precision, 4) == 0.9444)
        assert (round(hyp2_unigram_precision, 4) == 0.5714)

        # Bigram precision
        hyp1_bigram_precision = float(modified_precision(references, hyp1, n=2))
        hyp2_bigram_precision = float(modified_precision(references, hyp2, n=2))
        # Test bigram precision with assertAlmostEqual at 4 place precision.
        self.assertAlmostEqual(hyp1_bigram_precision, 0.58823529, places=4)
        self.assertAlmostEqual(hyp2_bigram_precision, 0.07692307, places=4)
        # Test bigram precision with rounding.
        assert (round(hyp1_bigram_precision, 4) == 0.5882)
        assert (round(hyp2_bigram_precision, 4) == 0.0769)

    def test_brevity_penalty(self):
        # Test case from brevity_penalty_closest function in mteval-v13a.pl.
        # Same test cases as in the doctest in nltk.translate.bleu_score.py
        references = [['a'] * 11, ['a'] * 8]
        hypothesis = ['a'] * 7
        hyp_len = len(hypothesis)
        closest_ref_len =  closest_ref_length(references, hyp_len)
        self.assertAlmostEqual(brevity_penalty(closest_ref_len, hyp_len), 0.8669, places=4)

        references = [['a'] * 11, ['a'] * 8, ['a'] * 6, ['a'] * 7]
        hypothesis = ['a'] * 7
        hyp_len = len(hypothesis)
        closest_ref_len =  closest_ref_length(references, hyp_len)
        assert brevity_penalty(closest_ref_len, hyp_len) == 1.0

    def test_zero_matches(self):
        # Test case where there's 0 matches
        references = ['The candidate has no alignment to any of the references'.split()]
        hypothesis = 'John loves Mary'.split()

        # Test BLEU to nth order of n-grams, where n is len(hypothesis).
        for n in range(1,len(hypothesis)):
            weights = [1.0/n] * n # Uniform weights.
            assert(sentence_bleu(references, hypothesis, weights) == 0)

    def test_full_matches(self):
        # Test case where there's 100% matches
        references = ['John loves Mary'.split()]
        hypothesis = 'John loves Mary'.split()

        # Test BLEU to nth order of n-grams, where n is len(hypothesis).
        for n in range(1,len(hypothesis)):
            weights = [1.0/n] * n # Uniform weights.
            assert(sentence_bleu(references, hypothesis, weights) == 1.0)

    def test_partial_matches_hypothesis_longer_than_reference(self):
        references = ['John loves Mary'.split()]
        hypothesis = 'John loves Mary who loves Mike'.split()
        # Since no 4-grams matches were found the result should be zero
        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.0, places=4)
        # Checks that the warning has been raised because len(reference) < 4.
        try:
            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
        except AttributeError:
            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.

#@unittest.skip("Skipping fringe cases for BLEU.")
class TestBLEUFringeCases(unittest.TestCase):

    def test_case_where_n_is_bigger_than_hypothesis_length(self):
        # Test BLEU to nth order of n-grams, where n > len(hypothesis).
        references = ['John loves Mary ?'.split()]
        hypothesis = 'John loves Mary'.split()
        n = len(hypothesis) + 1 #
        weights = [1.0/n] * n # Uniform weights.
        # Since no n-grams matches were found the result should be zero
        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
        self.assertAlmostEqual(sentence_bleu(references, hypothesis, weights), 0.0, places=4)
        # Checks that the warning has been raised because len(hypothesis) < 4.
        try:
            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
        except AttributeError:
            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.

        # Test case where n > len(hypothesis) but so is n > len(reference), and
        # it's a special case where reference == hypothesis.
        references = ['John loves Mary'.split()]
        hypothesis = 'John loves Mary'.split()
        # Since no 4-grams matches were found the result should be zero
        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
        self.assertAlmostEqual(sentence_bleu(references, hypothesis, weights), 0.0, places=4)

    def test_empty_hypothesis(self):
        # Test case where there's hypothesis is empty.
        references = ['The candidate has no alignment to any of the references'.split()]
        hypothesis = []
        assert(sentence_bleu(references, hypothesis) == 0)

    def test_empty_references(self):
        # Test case where there's reference is empty.
        references = [[]]
        hypothesis = 'John loves Mary'.split()
        assert(sentence_bleu(references, hypothesis) == 0)

    def test_empty_references_and_hypothesis(self):
        # Test case where both references and hypothesis is empty.
        references = [[]]
        hypothesis = []
        assert(sentence_bleu(references, hypothesis) == 0)

    def test_reference_or_hypothesis_shorter_than_fourgrams(self):
        # Tese case where the length of reference or hypothesis
        # is shorter than 4.
        references = ['let it go'.split()]
        hypothesis = 'let go it'.split()
        # Checks that the value the hypothesis and reference returns is 0.0
        # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.0, places=4)
        # Checks that the warning has been raised.
        try:
            self.assertWarns(UserWarning, sentence_bleu, references, hypothesis)
        except AttributeError:
            pass # unittest.TestCase.assertWarns is only supported in Python >= 3.2.

class TestBLEUvsMteval13a(unittest.TestCase):

    def test_corpus_bleu(self):
        ref_file = find('models/wmt15_eval/ref.ru')
        hyp_file = find('models/wmt15_eval/google.ru')
        mteval_output_file = find('models/wmt15_eval/mteval-13a.output')

        # Reads the BLEU scores from the `mteval-13a.output` file.
        # The order of the list corresponds to the order of the ngrams.
        with open(mteval_output_file, 'r') as mteval_fin:
            # The numbers are located in the last 2nd line of the file.
            # The first and 2nd item in the list are the score and system names.
            mteval_bleu_scores = map(float, mteval_fin.readlines()[-2].split()[1:-1])

        with io.open(ref_file, 'r', encoding='utf8') as ref_fin:
            with io.open(hyp_file, 'r', encoding='utf8') as hyp_fin:
                # Whitespace tokenize the file.
                # Note: split() automatically strip().
                hypothesis = list(map(lambda x: x.split(), hyp_fin))
                # Note that the corpus_bleu input is list of list of references.
                references = list(map(lambda x: [x.split()], ref_fin))
                # Without smoothing.
                for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
                    nltk_bleu = corpus_bleu(references, hypothesis, weights=(1.0/i,)*i)
                    # Check that the BLEU scores difference is less than 0.005 .
                    # Note: This is an approximate comparison; as much as
                    #       +/- 0.01 BLEU might be "statistically significant",
                    #       the actual translation quality might not be.
                    assert abs(mteval_bleu - nltk_bleu) < 0.005

                # With the same smoothing method used in mteval-v13a.pl
                chencherry = SmoothingFunction()
                for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
                    nltk_bleu = corpus_bleu(references, hypothesis,
                                            weights=(1.0/i,)*i,
                                            smoothing_function=chencherry.method3)
                    assert abs(mteval_bleu - nltk_bleu) < 0.005

class TestBLEUWithBadSentence(unittest.TestCase):
    def test_corpus_bleu_with_bad_sentence(self):
        hyp = "Teo S yb , oe uNb , R , T t , , t Tue Ar saln S , , 5istsi l , 5oe R ulO sae oR R"
        ref = str("Their tasks include changing a pump on the faulty stokehold ."
                  "Likewise , two species that are very similar in morphology "
                  "were distinguished using genetics .")
        references = [[ref.split()]]
        hypotheses = [hyp.split()]
        try: # Check that the warning is raised since no. of 2-grams < 0.
            with self.assertWarns(UserWarning):
                # Verify that the BLEU output is undesired since no. of 2-grams < 0.
                self.assertAlmostEqual(corpus_bleu(references, hypotheses), 0.0, places=4)
        except AttributeError: # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
            self.assertAlmostEqual(corpus_bleu(references, hypotheses), 0.0, places=4)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import calendar
import datetime
import logging

from django.apps import apps
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models, transaction
from django.utils import timezone
from django.utils.encoding import python_2_unicode_compatible
from django.utils.lru_cache import lru_cache

from jsonfield import JSONField
from model_utils import FieldTracker
from model_utils.models import TimeStampedModel

from nodeconductor.core import models as core_models
from nodeconductor.core.utils import hours_in_month
from nodeconductor.cost_tracking import managers, ConsumableItem
from nodeconductor.logging.loggers import LoggableMixin
from nodeconductor.logging.models import AlertThresholdMixin
from nodeconductor.structure import models as structure_models, SupportedServices

logger = logging.getLogger(__name__)

class EstimateUpdateError(Exception):
    pass

@python_2_unicode_compatible
class PriceEstimate(LoggableMixin, AlertThresholdMixin, core_models.UuidMixin, core_models.DescendantMixin):
    """ Store prices based on both estimates and actual consumption.

        Every record holds a list of children estimates.
                   /--- Service ---\
        Customer --                 ---> SPL --> Resource
                   \--- Project ---/
        Only resource node has actual data.
        Another ones should be re-calculated on every change of resource estimate.
    """
    content_type = models.ForeignKey(ContentType, null=True, related_name='+')
    object_id = models.PositiveIntegerField(null=True)
    scope = GenericForeignKey('content_type', 'object_id')
    details = JSONField(default={}, help_text='Saved scope details. Field is populated on scope deletion.')
    parents = models.ManyToManyField('PriceEstimate', related_name='children', help_text='Price estimate parents')

    total = models.FloatField(default=0, help_text='Predicted price for scope for current month.')
    consumed = models.FloatField(default=0, help_text='Price for resource until now.')
    limit = models.FloatField(
        default=-1, help_text='How many funds object can consume in current month."-1" means no limit.')

    month = models.PositiveSmallIntegerField(validators=[MaxValueValidator(12), MinValueValidator(1)])
    year = models.PositiveSmallIntegerField()

    objects = managers.PriceEstimateManager('scope')

    class Meta:
        unique_together = ('content_type', 'object_id', 'month', 'year',)

    def __str__(self):
        name = self._get_scope_name() if self.scope else self.details.get('name')
        return '%s for %s-%s %.2f' % (name, self.year, self.month, self.total)

    @classmethod
    @lru_cache(maxsize=1)
    def get_estimated_models(cls):
        return (
            structure_models.ResourceMixin.get_all_models() +
            structure_models.ServiceProjectLink.get_all_models() +
            structure_models.Service.get_all_models() +
            [structure_models.ServiceSettings] +
            [structure_models.Project, structure_models.Customer]
        )

    def get_parents(self):  # For DescendantMixin
        return self.parents.all()

    def get_children(self):  # For DescendantMixin
        return self.children.all()

    def get_log_fields(self):  # For LoggableMixin
        return 'uuid', 'scope', 'threshold', 'total', 'consumed'

    def is_over_threshold(self):  # For AlertThresholdMixin
        return self.total > self.threshold

    @classmethod
    def get_checkable_objects(cls):  # For AlertThresholdMixin
        """ Raise alerts only for price estimates that describes current month. """
        today = timezone.now()
        return cls.objects.filter(year=today.year, month=today.month)

    def get_previous(self):
        """ Get estimate for the same scope for previous month. """
        month, year = (self.month - 1, self.year) if self.month != 1 else (12, self.year - 1)
        return PriceEstimate.objects.get(scope=self.scope, month=month, year=year)

    def create_ancestors(self):
        """ Crete price estimates for scope ancestors if they does not exists """
        if not isinstance(self.scope, core_models.DescendantMixin):
            return
        scope_parents = self.scope.get_parents()
        for scope_parent in scope_parents:
            parent, created = PriceEstimate.objects.get_or_create(scope=scope_parent, month=self.month, year=self.year)
            self.parents.add(parent)
            if created:
                parent.create_ancestors()

    def init_details(self):
        """ Initialize price estimate details based on its scope """
        self.details = {
            'name': self._get_scope_name(),
            'description': getattr(self.scope, 'description', ''),
        }
        if hasattr(self.scope, 'backend_id'):
            self.details['backend_id'] = self.scope.backend_id
        self.save(update_fields=['details'])

    def update_total(self, update_ancestors=True):
        """ Re-calculate price of resource and its ancestors for whole month,
            based on its configuration and consumption details.
        """
        self._check_is_updatable()
        new_total = self._get_price(self.consumption_details.consumed_in_month)
        diff = new_total - self.total
        with transaction.atomic():
            self.total = new_total
            self.save(update_fields=['total'])
            if update_ancestors:
                self.update_ancestors_total(diff)

    def update_ancestors_total(self, diff):
        for ancestor in self.get_ancestors():
            ancestor.total += diff
            ancestor.save(update_fields=['total'])

    def update_consumed(self):
        """ Re-calculate price of resource until now. Does not update ancestors. """
        self._check_is_updatable()
        self.consumed = self._get_price(self.consumption_details.consumed_until_now)
        self.save(update_fields=['consumed'])

    def _get_price(self, consumed):
        """ Calculate price estimate for scope depends on consumed data and price list items.
            Map each consumable to price list item and multiply price its price by time of usage.
        """
        price_list_items = PriceListItem.get_for_resource(self.scope)
        consumables_prices = {(item.item_type, item.key): item.minute_rate for item in price_list_items}
        total = 0
        for consumable_item, usage in consumed.items():
            try:
                total += consumables_prices[(consumable_item.item_type, consumable_item.key)] * usage
            except KeyError:
                logger.error('Price list item for consumable "%s" does not exist.' % consumable_item)
        return total

    def _check_is_updatable(self):
        """ Raise error if price estimate does not have consumption details or
            does not belong to resource
        """
        if self.consumption_details is None:
            raise EstimateUpdateError('Cannot update consumed for price estimate that does not have consumption details.')
        if not isinstance(self.scope, structure_models.ResourceMixin):
            raise EstimateUpdateError('Cannot update consumed for price estimate that is not related to resource.')

    def _get_scope_name(self):
        if isinstance(self.scope, structure_models.ServiceProjectLink):
            # We need to display some meaningful name for SPL.
            return str(self.scope)
        else:
            return self.scope.name

class ConsumptionDetailUpdateError(Exception):
    pass

class ConsumptionDetailCalculateError(Exception):
    pass

class ConsumableItemsField(JSONField):
    """ Store consumable items and their usage as JSON.

        Represent data in format:
        {
            <ConsumableItem instance>: <usage>,
            <ConsumableItem instance>: <usage>,
            ...
        }
        Store data in format:
        [
            {
                "item_type": xx,
                "key": xx,
                "usage": xx,
            }
            ...
        ]
    """
    def pre_init(self, value, obj):
        """ JSON field initializes field in "pre_init" method, so it is better to override it. """
        value = super(ConsumableItemsField, self).pre_init(value, obj)
        if obj._state.adding:
            value = self._deserialize(value)
        return value

    def get_db_prep_value(self, value, connection, prepared=False):
        if prepared:
            return value

        if not isinstance(value, dict):
            raise TypeError('ConsumableItemsField value should be dict. Received: %s' % value)
        if any([not isinstance(item, ConsumableItem) for item in value]):
            raise TypeError('ConsumableItemsField keys should be instances of ConsumableItem class.')

        prep_value = self._serialize(value)
        return super(ConsumableItemsField, self).get_db_prep_value(prep_value, connection, prepared)

    def _serialize(self, value):
        return [{'usage': usage, 'item_type': item.item_type, 'key': item.key}
                for item, usage in value.items()]

    def _deserialize(self, serialized_value):
        return {ConsumableItem(item['item_type'], item['key']): item['usage'] for item in serialized_value}

class ConsumptionDetails(core_models.UuidMixin, TimeStampedModel):
    """ Resource consumption details per month.

        Warning! Use method "update_configuration" to update configurations,
        do not update them manually.
    """
    price_estimate = models.OneToOneField(PriceEstimate, related_name='consumption_details')
    configuration = ConsumableItemsField(default={}, help_text='Current resource configuration.')
    last_update_time = models.DateTimeField(help_text='Last configuration change time.')
    consumed_before_update = ConsumableItemsField(
        default={}, help_text='How many consumables were used by resource before last update.')

    objects = managers.ConsumptionDetailsManager()

    def update_configuration(self, new_configuration):
        """ Save how much consumables were used and update current configuration. """
        if new_configuration == self.configuration:
            return
        now = timezone.now()
        if now.month != self.price_estimate.month:
            raise ConsumptionDetailUpdateError('It is possible to update consumption details only for current month.')
        minutes_from_last_update = self._get_minutes_from_last_update(now)
        for consumable_item, usage in self.configuration.items():
            consumed_after_modification = usage * minutes_from_last_update
            self.consumed_before_update[consumable_item] = (
                self.consumed_before_update.get(consumable_item, 0) + consumed_after_modification)
        self.configuration = new_configuration
        self.last_update_time = now
        self.save()

    @property
    def consumed_in_month(self):
        """ How many resources were (or will be) consumed until end of the month """
        return self._get_consumed(self._get_month_end())

    @property
    def consumed_until_now(self):
        """ How many consumables were used by resource until now. """
        return self._get_consumed(timezone.now())

    def _get_consumed(self, time):
        """ How many consumables were used (or will be) by resource until given time. """
        minutes_from_last_update = self._get_minutes_from_last_update(time)
        if minutes_from_last_update < 0:
            raise ConsumptionDetailCalculateError('Cannot calculate consumption if time < last modification date.')
        _consumed = {}
        for consumable_item in set(self.configuration.keys() + self.consumed_before_update.keys()):
            after_update = self.configuration.get(consumable_item, 0) * minutes_from_last_update
            before_update = self.consumed_before_update.get(consumable_item, 0)
            _consumed[consumable_item] = after_update + before_update
        return _consumed

    def _get_month_end(self):
        year, month = self.price_estimate.year, self.price_estimate.month
        days_in_month = calendar.monthrange(year, month)[1]
        last_day_of_month = datetime.date(month=month, year=year, day=days_in_month)
        last_second_of_month = datetime.datetime.combine(last_day_of_month, datetime.time.max)
        return timezone.make_aware(last_second_of_month, timezone.get_current_timezone())

    def _get_minutes_from_last_update(self, time):
        """ How much minutes passed from last update to given time """
        time_from_last_update = time - self.last_update_time
        return int(time_from_last_update.total_seconds() / 60)

class AbstractPriceListItem(models.Model):
    class Meta:
        abstract = True

    value = models.DecimalField("Hourly rate", default=0, max_digits=11, decimal_places=5)
    units = models.CharField(max_length=255, blank=True)

    @property
    def monthly_rate(self):
        return '%0.2f' % (self.value * 60 * hours_in_month())

    @property
    def minute_rate(self):
        return float(self.value) / 60

@python_2_unicode_compatible
class DefaultPriceListItem(core_models.UuidMixin, core_models.NameMixin, AbstractPriceListItem):
    """
    Default price list item for all resources of supported service types.

    It is fetched from cost tracking backend.
    Field "name" represents how price item will be represented for user.
    """
    item_type = models.CharField(max_length=255, help_text='Type of price list item. Examples: storage, flavor.')
    key = models.CharField(
        max_length=255, help_text='Key that corresponds particular consumable. Example: name of flavor.')
    resource_content_type = models.ForeignKey(ContentType, default=None)
    # Field "metadata" is deprecated. We decided to store objects separately from their prices.
    metadata = JSONField(
        blank=True, help_text='Details of the item, that corresponds price list item. Example: details of flavor.')

    tracker = FieldTracker()

    def __str__(self):
        return 'Price list item %s: %s = %s for %s' % (self.name, self.key, self.value, self.resource_content_type)

    class Meta:
        unique_together = ('key', 'item_type', 'resource_content_type')

    @property
    def resource_type(self):
        cls = self.resource_content_type.model_class()
        if cls:
            return SupportedServices.get_name_for_model(cls)

class PriceListItem(core_models.UuidMixin, AbstractPriceListItem):
    """
    Price list item related to private service.
    It is entered manually by customer owner.
    """
    # Generic key to service
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    service = GenericForeignKey('content_type', 'object_id')
    objects = managers.PriceListItemManager('service')
    default_price_list_item = models.ForeignKey(DefaultPriceListItem)

    class Meta:
        unique_together = ('content_type', 'object_id', 'default_price_list_item')

    def clean(self):
        if SupportedServices.is_public_service(self.service):
            raise ValidationError('Public service does not support price list items')

        resource = self.default_price_list_item.resource_content_type.model_class()
        valid_resources = SupportedServices.get_related_models(self.service)['resources']

        if resource not in valid_resources:
            raise ValidationError('Service does not support required content type')

    @staticmethod
    def get_for_resource(resource):
        """ Get list of all price list items that should be used for resource.

            If price list item is defined for service - return it, otherwise -
            return default price list item.
        """
        resource_content_type = ContentType.objects.get_for_model(resource)
        default_items = set(DefaultPriceListItem.objects.filter(resource_content_type=resource_content_type))
        service = resource.service_project_link.service
        items = set(PriceListItem.objects.filter(
            default_price_list_item__in=default_items, service=service).select_related('default_price_list_item'))
        rewrited_defaults = set([i.default_price_list_item for i in items])
        return items | (default_items - rewrited_defaults)

# Deprecated. Should be removed.
class PayableMixin(models.Model):
    """ Extend Resource model with methods to track usage cost and handle orders """

    billing_backend_id = models.CharField(max_length=255, blank=True, help_text='ID of a resource in backend')
    last_usage_update_time = models.DateTimeField(blank=True, null=True)

    @classmethod
    @lru_cache(maxsize=1)
    def get_all_models(cls):
        return [model for model in apps.get_models() if issubclass(model, cls)]

    class Meta(object):
        abstract = True
<commit_msg>Fix deserialization check<commit_after>from __future__ import unicode_literals

import calendar
import datetime
import logging

from django.apps import apps
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models, transaction
from django.utils import timezone
from django.utils.encoding import python_2_unicode_compatible
from django.utils.lru_cache import lru_cache

from jsonfield import JSONField
from model_utils import FieldTracker
from model_utils.models import TimeStampedModel

from nodeconductor.core import models as core_models
from nodeconductor.core.utils import hours_in_month
from nodeconductor.cost_tracking import managers, ConsumableItem
from nodeconductor.logging.loggers import LoggableMixin
from nodeconductor.logging.models import AlertThresholdMixin
from nodeconductor.structure import models as structure_models, SupportedServices

logger = logging.getLogger(__name__)

class EstimateUpdateError(Exception):
    pass

@python_2_unicode_compatible
class PriceEstimate(LoggableMixin, AlertThresholdMixin, core_models.UuidMixin, core_models.DescendantMixin):
    """ Store prices based on both estimates and actual consumption.

        Every record holds a list of children estimates.
                   /--- Service ---\
        Customer --                 ---> SPL --> Resource
                   \--- Project ---/
        Only resource node has actual data.
        Another ones should be re-calculated on every change of resource estimate.
    """
    content_type = models.ForeignKey(ContentType, null=True, related_name='+')
    object_id = models.PositiveIntegerField(null=True)
    scope = GenericForeignKey('content_type', 'object_id')
    details = JSONField(default={}, help_text='Saved scope details. Field is populated on scope deletion.')
    parents = models.ManyToManyField('PriceEstimate', related_name='children', help_text='Price estimate parents')

    total = models.FloatField(default=0, help_text='Predicted price for scope for current month.')
    consumed = models.FloatField(default=0, help_text='Price for resource until now.')
    limit = models.FloatField(
        default=-1, help_text='How many funds object can consume in current month."-1" means no limit.')

    month = models.PositiveSmallIntegerField(validators=[MaxValueValidator(12), MinValueValidator(1)])
    year = models.PositiveSmallIntegerField()

    objects = managers.PriceEstimateManager('scope')

    class Meta:
        unique_together = ('content_type', 'object_id', 'month', 'year',)

    def __str__(self):
        name = self._get_scope_name() if self.scope else self.details.get('name')
        return '%s for %s-%s %.2f' % (name, self.year, self.month, self.total)

    @classmethod
    @lru_cache(maxsize=1)
    def get_estimated_models(cls):
        return (
            structure_models.ResourceMixin.get_all_models() +
            structure_models.ServiceProjectLink.get_all_models() +
            structure_models.Service.get_all_models() +
            [structure_models.ServiceSettings] +
            [structure_models.Project, structure_models.Customer]
        )

    def get_parents(self):  # For DescendantMixin
        return self.parents.all()

    def get_children(self):  # For DescendantMixin
        return self.children.all()

    def get_log_fields(self):  # For LoggableMixin
        return 'uuid', 'scope', 'threshold', 'total', 'consumed'

    def is_over_threshold(self):  # For AlertThresholdMixin
        return self.total > self.threshold

    @classmethod
    def get_checkable_objects(cls):  # For AlertThresholdMixin
        """ Raise alerts only for price estimates that describes current month. """
        today = timezone.now()
        return cls.objects.filter(year=today.year, month=today.month)

    def get_previous(self):
        """ Get estimate for the same scope for previous month. """
        month, year = (self.month - 1, self.year) if self.month != 1 else (12, self.year - 1)
        return PriceEstimate.objects.get(scope=self.scope, month=month, year=year)

    def create_ancestors(self):
        """ Crete price estimates for scope ancestors if they does not exists """
        if not isinstance(self.scope, core_models.DescendantMixin):
            return
        scope_parents = self.scope.get_parents()
        for scope_parent in scope_parents:
            parent, created = PriceEstimate.objects.get_or_create(scope=scope_parent, month=self.month, year=self.year)
            self.parents.add(parent)
            if created:
                parent.create_ancestors()

    def init_details(self):
        """ Initialize price estimate details based on its scope """
        self.details = {
            'name': self._get_scope_name(),
            'description': getattr(self.scope, 'description', ''),
        }
        if hasattr(self.scope, 'backend_id'):
            self.details['backend_id'] = self.scope.backend_id
        self.save(update_fields=['details'])

    def update_total(self, update_ancestors=True):
        """ Re-calculate price of resource and its ancestors for whole month,
            based on its configuration and consumption details.
        """
        self._check_is_updatable()
        new_total = self._get_price(self.consumption_details.consumed_in_month)
        diff = new_total - self.total
        with transaction.atomic():
            self.total = new_total
            self.save(update_fields=['total'])
            if update_ancestors:
                self.update_ancestors_total(diff)

    def update_ancestors_total(self, diff):
        for ancestor in self.get_ancestors():
            ancestor.total += diff
            ancestor.save(update_fields=['total'])

    def update_consumed(self):
        """ Re-calculate price of resource until now. Does not update ancestors. """
        self._check_is_updatable()
        self.consumed = self._get_price(self.consumption_details.consumed_until_now)
        self.save(update_fields=['consumed'])

    def _get_price(self, consumed):
        """ Calculate price estimate for scope depends on consumed data and price list items.
            Map each consumable to price list item and multiply price its price by time of usage.
        """
        price_list_items = PriceListItem.get_for_resource(self.scope)
        consumables_prices = {(item.item_type, item.key): item.minute_rate for item in price_list_items}
        total = 0
        for consumable_item, usage in consumed.items():
            try:
                total += consumables_prices[(consumable_item.item_type, consumable_item.key)] * usage
            except KeyError:
                logger.error('Price list item for consumable "%s" does not exist.' % consumable_item)
        return total

    def _check_is_updatable(self):
        """ Raise error if price estimate does not have consumption details or
            does not belong to resource
        """
        if self.consumption_details is None:
            raise EstimateUpdateError('Cannot update consumed for price estimate that does not have consumption details.')
        if not isinstance(self.scope, structure_models.ResourceMixin):
            raise EstimateUpdateError('Cannot update consumed for price estimate that is not related to resource.')

    def _get_scope_name(self):
        if isinstance(self.scope, structure_models.ServiceProjectLink):
            # We need to display some meaningful name for SPL.
            return str(self.scope)
        else:
            return self.scope.name

class ConsumptionDetailUpdateError(Exception):
    pass

class ConsumptionDetailCalculateError(Exception):
    pass

class ConsumableItemsField(JSONField):
    """ Store consumable items and their usage as JSON.

        Represent data in format:
        {
            <ConsumableItem instance>: <usage>,
            <ConsumableItem instance>: <usage>,
            ...
        }
        Store data in format:
        [
            {
                "item_type": xx,
                "key": xx,
                "usage": xx,
            }
            ...
        ]
    """
    def pre_init(self, value, obj):
        """ JSON field initializes field in "pre_init" method, so it is better to override it. """
        value = super(ConsumableItemsField, self).pre_init(value, obj)
        if isinstance(value, list):
            value = self._deserialize(value)
        return value

    def get_db_prep_value(self, value, connection, prepared=False):
        if prepared:
            return value

        if not isinstance(value, dict):
            raise TypeError('ConsumableItemsField value should be dict. Received: %s' % value)
        if any([not isinstance(item, ConsumableItem) for item in value]):
            raise TypeError('ConsumableItemsField keys should be instances of ConsumableItem class.')

        prep_value = self._serialize(value)
        return super(ConsumableItemsField, self).get_db_prep_value(prep_value, connection, prepared)

    def _serialize(self, value):
        return [{'usage': usage, 'item_type': item.item_type, 'key': item.key}
                for item, usage in value.items()]

    def _deserialize(self, serialized_value):
        return {ConsumableItem(item['item_type'], item['key']): item['usage'] for item in serialized_value}

class ConsumptionDetails(core_models.UuidMixin, TimeStampedModel):
    """ Resource consumption details per month.

        Warning! Use method "update_configuration" to update configurations,
        do not update them manually.
    """
    price_estimate = models.OneToOneField(PriceEstimate, related_name='consumption_details')
    configuration = ConsumableItemsField(default={}, help_text='Current resource configuration.')
    last_update_time = models.DateTimeField(help_text='Last configuration change time.')
    consumed_before_update = ConsumableItemsField(
        default={}, help_text='How many consumables were used by resource before last update.')

    objects = managers.ConsumptionDetailsManager()

    def update_configuration(self, new_configuration):
        """ Save how much consumables were used and update current configuration. """
        if new_configuration == self.configuration:
            return
        now = timezone.now()
        if now.month != self.price_estimate.month:
            raise ConsumptionDetailUpdateError('It is possible to update consumption details only for current month.')
        minutes_from_last_update = self._get_minutes_from_last_update(now)
        for consumable_item, usage in self.configuration.items():
            consumed_after_modification = usage * minutes_from_last_update
            self.consumed_before_update[consumable_item] = (
                self.consumed_before_update.get(consumable_item, 0) + consumed_after_modification)
        self.configuration = new_configuration
        self.last_update_time = now
        self.save()

    @property
    def consumed_in_month(self):
        """ How many resources were (or will be) consumed until end of the month """
        return self._get_consumed(self._get_month_end())

    @property
    def consumed_until_now(self):
        """ How many consumables were used by resource until now. """
        return self._get_consumed(timezone.now())

    def _get_consumed(self, time):
        """ How many consumables were used (or will be) by resource until given time. """
        minutes_from_last_update = self._get_minutes_from_last_update(time)
        if minutes_from_last_update < 0:
            raise ConsumptionDetailCalculateError('Cannot calculate consumption if time < last modification date.')
        _consumed = {}
        for consumable_item in set(self.configuration.keys() + self.consumed_before_update.keys()):
            after_update = self.configuration.get(consumable_item, 0) * minutes_from_last_update
            before_update = self.consumed_before_update.get(consumable_item, 0)
            _consumed[consumable_item] = after_update + before_update
        return _consumed

    def _get_month_end(self):
        year, month = self.price_estimate.year, self.price_estimate.month
        days_in_month = calendar.monthrange(year, month)[1]
        last_day_of_month = datetime.date(month=month, year=year, day=days_in_month)
        last_second_of_month = datetime.datetime.combine(last_day_of_month, datetime.time.max)
        return timezone.make_aware(last_second_of_month, timezone.get_current_timezone())

    def _get_minutes_from_last_update(self, time):
        """ How much minutes passed from last update to given time """
        time_from_last_update = time - self.last_update_time
        return int(time_from_last_update.total_seconds() / 60)

class AbstractPriceListItem(models.Model):
    class Meta:
        abstract = True

    value = models.DecimalField("Hourly rate", default=0, max_digits=11, decimal_places=5)
    units = models.CharField(max_length=255, blank=True)

    @property
    def monthly_rate(self):
        return '%0.2f' % (self.value * 60 * hours_in_month())

    @property
    def minute_rate(self):
        return float(self.value) / 60

@python_2_unicode_compatible
class DefaultPriceListItem(core_models.UuidMixin, core_models.NameMixin, AbstractPriceListItem):
    """
    Default price list item for all resources of supported service types.

    It is fetched from cost tracking backend.
    Field "name" represents how price item will be represented for user.
    """
    item_type = models.CharField(max_length=255, help_text='Type of price list item. Examples: storage, flavor.')
    key = models.CharField(
        max_length=255, help_text='Key that corresponds particular consumable. Example: name of flavor.')
    resource_content_type = models.ForeignKey(ContentType, default=None)
    # Field "metadata" is deprecated. We decided to store objects separately from their prices.
    metadata = JSONField(
        blank=True, help_text='Details of the item, that corresponds price list item. Example: details of flavor.')

    tracker = FieldTracker()

    def __str__(self):
        return 'Price list item %s: %s = %s for %s' % (self.name, self.key, self.value, self.resource_content_type)

    class Meta:
        unique_together = ('key', 'item_type', 'resource_content_type')

    @property
    def resource_type(self):
        cls = self.resource_content_type.model_class()
        if cls:
            return SupportedServices.get_name_for_model(cls)

class PriceListItem(core_models.UuidMixin, AbstractPriceListItem):
    """
    Price list item related to private service.
    It is entered manually by customer owner.
    """
    # Generic key to service
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    service = GenericForeignKey('content_type', 'object_id')
    objects = managers.PriceListItemManager('service')
    default_price_list_item = models.ForeignKey(DefaultPriceListItem)

    class Meta:
        unique_together = ('content_type', 'object_id', 'default_price_list_item')

    def clean(self):
        if SupportedServices.is_public_service(self.service):
            raise ValidationError('Public service does not support price list items')

        resource = self.default_price_list_item.resource_content_type.model_class()
        valid_resources = SupportedServices.get_related_models(self.service)['resources']

        if resource not in valid_resources:
            raise ValidationError('Service does not support required content type')

    @staticmethod
    def get_for_resource(resource):
        """ Get list of all price list items that should be used for resource.

            If price list item is defined for service - return it, otherwise -
            return default price list item.
        """
        resource_content_type = ContentType.objects.get_for_model(resource)
        default_items = set(DefaultPriceListItem.objects.filter(resource_content_type=resource_content_type))
        service = resource.service_project_link.service
        items = set(PriceListItem.objects.filter(
            default_price_list_item__in=default_items, service=service).select_related('default_price_list_item'))
        rewrited_defaults = set([i.default_price_list_item for i in items])
        return items | (default_items - rewrited_defaults)

# Deprecated. Should be removed.
class PayableMixin(models.Model):
    """ Extend Resource model with methods to track usage cost and handle orders """

    billing_backend_id = models.CharField(max_length=255, blank=True, help_text='ID of a resource in backend')
    last_usage_update_time = models.DateTimeField(blank=True, null=True)

    @classmethod
    @lru_cache(maxsize=1)
    def get_all_models(cls):
        return [model for model in apps.get_models() if issubclass(model, cls)]

    class Meta(object):
        abstract = True
<|endoftext|>


<commit_before>__version__ = "0.9.1"
<commit_msg>minor cleanup, win32 packaging tested, begin major cleanup<commit_after>__version__ = "0.9.2"
<|endoftext|>


<commit_before>from flask.wrappers import Request
from flask import request, current_app, abort

class NotifyRequest(Request):
    """
        A custom Request class, implementing extraction of zipkin headers used to trace request through cloudfoundry
        as described here: https://docs.cloudfoundry.org/concepts/http-routing.html#zipkin-headers
    """

    @property
    def request_id(self):
        return self.trace_id

    @property
    def trace_id(self):
        """
            The "trace id" (in zipkin terms) assigned to this request, if present (None otherwise)
        """
        if not hasattr(self, "_trace_id"):
            self._trace_id = self._get_header_value(current_app.config['NOTIFY_TRACE_ID_HEADER'])
        return self._trace_id

    @property
    def span_id(self):
        """
            The "span id" (in zipkin terms) set in this request's header, if present (None otherwise)
        """
        if not hasattr(self, "_span_id"):
            # note how we don't generate an id of our own. not being supplied a span id implies that we are running in
            # an environment with no span-id-aware request router, and thus would have no intermediary to prevent the
            # propagation of our span id all the way through all our onwards requests much like trace id. and the point
            # of span id is to assign identifiers to each individual request.
            self._span_id = self._get_header_value(current_app.config['NOTIFY_SPAN_ID_HEADER'])
        return self._span_id

    @property
    def parent_span_id(self):
        """
            The "parent span id" (in zipkin terms) set in this request's header, if present (None otherwise)
        """
        if not hasattr(self, "_parent_span_id"):
            self._parent_span_id = self._get_header_value(current_app.config['NOTIFY_PARENT_SPAN_ID_HEADER'])
        return self._parent_span_id

    def _get_header_value(self, header_name):
        """
        Returns value of the given header
        """
        if header_name in self.headers and self.headers[header_name]:
            return self.headers[header_name]

        return None

class ResponseHeaderMiddleware(object):
    def __init__(self, app, trace_id_header, span_id_header):
        self.app = app
        self.trace_id_header = trace_id_header
        self.span_id_header = span_id_header

    def __call__(self, environ, start_response):
        def rewrite_response_headers(status, headers, exc_info=None):
            lower_existing_header_names = frozenset(name.lower() for name, value in headers)

            if self.trace_id_header not in lower_existing_header_names:
                headers.append((self.trace_id_header, str(request.trace_id)))

            if self.span_id_header not in lower_existing_header_names:
                headers.append((self.span_id_header, str(request.span_id)))

            return start_response(status, headers, exc_info)

        return self.app(environ, rewrite_response_headers)

def init_app(app):
    app.config.setdefault("NOTIFY_TRACE_ID_HEADER", "X-B3-TraceId")
    app.config.setdefault("NOTIFY_SPAN_ID_HEADER", "X-B3-SpanId")
    app.config.setdefault("NOTIFY_PARENT_SPAN_ID_HEADER", "X-B3-ParentSpanId")

    app.request_class = NotifyRequest
    app.wsgi_app = ResponseHeaderMiddleware(
        app.wsgi_app,
        app.config['NOTIFY_TRACE_ID_HEADER'],
        app.config['NOTIFY_SPAN_ID_HEADER'],
    )

def check_proxy_header_before_request():
    keys = [
        current_app.config.get('ROUTE_SECRET_KEY_1'),
        current_app.config.get('ROUTE_SECRET_KEY_2'),
    ]
    result, msg = _check_proxy_header_secret(request, keys)

    if not result:
        current_app.logger.warning(msg)
        if current_app.config.get('CHECK_PROXY_HEADER', False):
            abort(403)

    # We need to return None to continue processing the request
    # http://flask.pocoo.org/docs/0.12/api/#flask.Flask.before_request
    return None

def _check_proxy_header_secret(request, secrets, header='X-Custom-Forwarder'):
    if header not in request.headers:
        return False, "Header missing"

    header_secret = request.headers.get(header)
    if not header_secret:
        return False, "Header exists but is empty"

    # if there isn't any non-empty secret configured we fail closed
    if not any(secrets):
        return False, "Secrets are not configured"

    for i, secret in enumerate(secrets):
        if header_secret == secret:
            return True, "Key used: {}".format(i + 1)  # add 1 to make it human-compatible

    return False, "Header didn't match any keys"
<commit_msg>Don't log proxy header warnings if the check is disabled<commit_after>from flask.wrappers import Request
from flask import request, current_app, abort

class NotifyRequest(Request):
    """
        A custom Request class, implementing extraction of zipkin headers used to trace request through cloudfoundry
        as described here: https://docs.cloudfoundry.org/concepts/http-routing.html#zipkin-headers
    """

    @property
    def request_id(self):
        return self.trace_id

    @property
    def trace_id(self):
        """
            The "trace id" (in zipkin terms) assigned to this request, if present (None otherwise)
        """
        if not hasattr(self, "_trace_id"):
            self._trace_id = self._get_header_value(current_app.config['NOTIFY_TRACE_ID_HEADER'])
        return self._trace_id

    @property
    def span_id(self):
        """
            The "span id" (in zipkin terms) set in this request's header, if present (None otherwise)
        """
        if not hasattr(self, "_span_id"):
            # note how we don't generate an id of our own. not being supplied a span id implies that we are running in
            # an environment with no span-id-aware request router, and thus would have no intermediary to prevent the
            # propagation of our span id all the way through all our onwards requests much like trace id. and the point
            # of span id is to assign identifiers to each individual request.
            self._span_id = self._get_header_value(current_app.config['NOTIFY_SPAN_ID_HEADER'])
        return self._span_id

    @property
    def parent_span_id(self):
        """
            The "parent span id" (in zipkin terms) set in this request's header, if present (None otherwise)
        """
        if not hasattr(self, "_parent_span_id"):
            self._parent_span_id = self._get_header_value(current_app.config['NOTIFY_PARENT_SPAN_ID_HEADER'])
        return self._parent_span_id

    def _get_header_value(self, header_name):
        """
        Returns value of the given header
        """
        if header_name in self.headers and self.headers[header_name]:
            return self.headers[header_name]

        return None

class ResponseHeaderMiddleware(object):
    def __init__(self, app, trace_id_header, span_id_header):
        self.app = app
        self.trace_id_header = trace_id_header
        self.span_id_header = span_id_header

    def __call__(self, environ, start_response):
        def rewrite_response_headers(status, headers, exc_info=None):
            lower_existing_header_names = frozenset(name.lower() for name, value in headers)

            if self.trace_id_header not in lower_existing_header_names:
                headers.append((self.trace_id_header, str(request.trace_id)))

            if self.span_id_header not in lower_existing_header_names:
                headers.append((self.span_id_header, str(request.span_id)))

            return start_response(status, headers, exc_info)

        return self.app(environ, rewrite_response_headers)

def init_app(app):
    app.config.setdefault("NOTIFY_TRACE_ID_HEADER", "X-B3-TraceId")
    app.config.setdefault("NOTIFY_SPAN_ID_HEADER", "X-B3-SpanId")
    app.config.setdefault("NOTIFY_PARENT_SPAN_ID_HEADER", "X-B3-ParentSpanId")

    app.request_class = NotifyRequest
    app.wsgi_app = ResponseHeaderMiddleware(
        app.wsgi_app,
        app.config['NOTIFY_TRACE_ID_HEADER'],
        app.config['NOTIFY_SPAN_ID_HEADER'],
    )

def check_proxy_header_before_request():
    keys = [
        current_app.config.get('ROUTE_SECRET_KEY_1'),
        current_app.config.get('ROUTE_SECRET_KEY_2'),
    ]
    result, msg = _check_proxy_header_secret(request, keys)

    if not result:
        if current_app.config.get('CHECK_PROXY_HEADER', False):
            current_app.logger.warning(msg)
            abort(403)

    # We need to return None to continue processing the request
    # http://flask.pocoo.org/docs/0.12/api/#flask.Flask.before_request
    return None

def _check_proxy_header_secret(request, secrets, header='X-Custom-Forwarder'):
    if header not in request.headers:
        return False, "Header missing"

    header_secret = request.headers.get(header)
    if not header_secret:
        return False, "Header exists but is empty"

    # if there isn't any non-empty secret configured we fail closed
    if not any(secrets):
        return False, "Secrets are not configured"

    for i, secret in enumerate(secrets):
        if header_secret == secret:
            return True, "Key used: {}".format(i + 1)  # add 1 to make it human-compatible

    return False, "Header didn't match any keys"
<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import
import os
import ctypes

import numpy as np

import numba.unittest_support as unittest
from numba.hsa.hsadrv.driver import hsa, Queue, BrigModule
from numba.hsa.hsadrv import drvapi, enums

class TestAgents(unittest.TestCase):
    def test_agents_init(self):
        self.assertGreater(len(hsa.agents), 0)

    def test_agents_create_queue_single(self):
        for agent in hsa.agents:
            queue = agent.create_queue_single(2 ** 5)
            self.assertIsInstance(queue, Queue)

    def test_agents_create_queue_multi(self):
        for agent in hsa.agents:
            queue = agent.create_queue_multi(2 ** 5)
            self.assertIsInstance(queue, Queue)

class _TestBase(unittest.TestCase):
    def setUp(self):
        self.gpu = [a for a in hsa.agents if a.is_component][0]
        self.cpu = [a for a in hsa.agents if not a.is_component][0]
        self.queue = self.gpu.create_queue_multi(self.gpu.queue_max_size)

    def tearDown(self):
        del self.queue
        del self.gpu
        del self.cpu

def get_brig_file():
    basedir = os.path.dirname(__file__)
    path = os.path.join(basedir, 'vector_copy.brig')
    assert os.path.isfile(path)
    return path

class TestBrigModule(unittest.TestCase):
    def test_from_file(self):
        brig_file = get_brig_file()
        brig_module = BrigModule.from_file(brig_file)
        offset = brig_module.find_symbol_offset('&__vector_copy_kernel')
        self.assertNotEqual(offset, 0)

class TestProgram(_TestBase):
    def test_create_program(self):
        brig_file = get_brig_file()
        symbol = '&__vector_copy_kernel'
        brig_module = BrigModule.from_file(brig_file)
        symbol_offset = brig_module.find_symbol_offset(symbol)
        program = hsa.create_program([self.gpu])
        module = program.add_module(brig_module)
        code_descriptor = program.finalize(self.gpu, module, symbol_offset)
        self.assertGreater(code_descriptor._id.kernarg_segment_byte_size, 0)

class TestMemory(_TestBase):
    def test_register(self):
        src = np.random.random(1024).astype(np.float32)
        hsa.hsa_memory_register(src.ctypes.data, src.nbytes)
        hsa.hsa_memory_deregister(src.ctypes.data, src.nbytes)

    def test_allocate(self):
        my_region = drvapi.hsa_region_t(0)

        def _callback(region, result):
            flags = drvapi.hsa_region_flag_t()
            hsa.hsa_region_get_info(region, enums.HSA_REGION_INFO_FLAGS,
                                    ctypes.byref(flags))
            if not (flags.value & enums.HSA_REGION_FLAG_KERNARG):
                result.value = region
            return enums.HSA_STATUS_SUCCESS

        callback = drvapi.HSA_AGENT_ITERATE_REGIONS_CALLBACK_FUNC(_callback)
        hsa.hsa_agent_iterate_regions(self.cpu._id, callback, my_region)

        self.assertNotEqual(my_region.value, 0)

        src = np.random.random(1024).astype(np.float32)

        ptr = ctypes.c_void_p(0)
        hsa.hsa_memory_allocate(my_region, src.nbytes, ctypes.byref(ptr))
        ctypes.memmove(ptr, src.ctypes.data, src.nbytes)

        data = ctypes.cast(ptr, ctypes.POINTER(ctypes.c_float))

        for i in range(src.size):
            self.assertEqual(data[i], src[i])

        hsa.hsa_memory_free(ptr)

if __name__ == '__main__':
    unittest.main()
<commit_msg>added test to check all API_PROTOTYPES bind<commit_after>from __future__ import print_function, absolute_import
import os
import ctypes

import numpy as np

import numba.unittest_support as unittest
from numba.hsa.hsadrv.driver import hsa, Queue, BrigModule
from numba.hsa.hsadrv import drvapi, enums

class TestLowLevelApi(unittest.Tests):
    """This test checks that all the functions defined in drvapi
    bind properly using ctypes."""

    def test_functions_available(self):
        missing_functions = []
        for fname in drvapi.API_PROTOTYPES.keys():
            try:
                getattr(hsa, fname)
            except Exception as e:
                missing_functions.append("'{0}': {1}".format(fname, str(e)))

        self.assertEqual(len(missing_functions), 0, msg='\n'.join(missing_functions))

class TestAgents(unittest.TestCase):
    def test_agents_init(self):
        self.assertGreater(len(hsa.agents), 0)

    def test_agents_create_queue_single(self):
        for agent in hsa.agents:
            queue = agent.create_queue_single(2 ** 5)
            self.assertIsInstance(queue, Queue)

    def test_agents_create_queue_multi(self):
        for agent in hsa.agents:
            queue = agent.create_queue_multi(2 ** 5)
            self.assertIsInstance(queue, Queue)

class _TestBase(unittest.TestCase):
    def setUp(self):
        self.gpu = [a for a in hsa.agents if a.is_component][0]
        self.cpu = [a for a in hsa.agents if not a.is_component][0]
        self.queue = self.gpu.create_queue_multi(self.gpu.queue_max_size)

    def tearDown(self):
        del self.queue
        del self.gpu
        del self.cpu

def get_brig_file():
    basedir = os.path.dirname(__file__)
    path = os.path.join(basedir, 'vector_copy.brig')
    assert os.path.isfile(path)
    return path

class TestBrigModule(unittest.TestCase):
    def test_from_file(self):
        brig_file = get_brig_file()
        brig_module = BrigModule.from_file(brig_file)
        offset = brig_module.find_symbol_offset('&__vector_copy_kernel')
        self.assertNotEqual(offset, 0)

class TestProgram(_TestBase):
    def test_create_program(self):
        brig_file = get_brig_file()
        symbol = '&__vector_copy_kernel'
        brig_module = BrigModule.from_file(brig_file)
        symbol_offset = brig_module.find_symbol_offset(symbol)
        program = hsa.create_program([self.gpu])
        module = program.add_module(brig_module)
        code_descriptor = program.finalize(self.gpu, module, symbol_offset)
        self.assertGreater(code_descriptor._id.kernarg_segment_byte_size, 0)

class TestMemory(_TestBase):
    def test_register(self):
        src = np.random.random(1024).astype(np.float32)
        hsa.hsa_memory_register(src.ctypes.data, src.nbytes)
        hsa.hsa_memory_deregister(src.ctypes.data, src.nbytes)

    def test_allocate(self):
        my_region = drvapi.hsa_region_t(0)

        def _callback(region, result):
            flags = drvapi.hsa_region_flag_t()
            hsa.hsa_region_get_info(region, enums.HSA_REGION_INFO_FLAGS,
                                    ctypes.byref(flags))
            if not (flags.value & enums.HSA_REGION_FLAG_KERNARG):
                result.value = region
            return enums.HSA_STATUS_SUCCESS

        callback = drvapi.HSA_AGENT_ITERATE_REGIONS_CALLBACK_FUNC(_callback)
        hsa.hsa_agent_iterate_regions(self.cpu._id, callback, my_region)

        self.assertNotEqual(my_region.value, 0)

        src = np.random.random(1024).astype(np.float32)

        ptr = ctypes.c_void_p(0)
        hsa.hsa_memory_allocate(my_region, src.nbytes, ctypes.byref(ptr))
        ctypes.memmove(ptr, src.ctypes.data, src.nbytes)

        data = ctypes.cast(ptr, ctypes.POINTER(ctypes.c_float))

        for i in range(src.size):
            self.assertEqual(data[i], src[i])

        hsa.hsa_memory_free(ptr)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>