<commit_before># Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

from st2common import log as logging

from st2common.constants.keyvalue import DATASTORE_PARENT_SCOPE
from st2common.constants.keyvalue import SYSTEM_SCOPE, FULL_SYSTEM_SCOPE
from st2common.constants.keyvalue import USER_SCOPE, FULL_USER_SCOPE
from st2common.constants.keyvalue import ALLOWED_SCOPES
from st2common.constants.keyvalue import DATASTORE_KEY_SEPARATOR
from st2common.exceptions.db import StackStormDBObjectNotFoundError
from st2common.exceptions.keyvalue import InvalidScopeException, InvalidUserException
from st2common.models.system.keyvalue import UserKeyReference
from st2common.persistence.keyvalue import KeyValuePair
from st2common.persistence.rbac import UserRoleAssignment
from st2common.persistence.rbac import Role
from st2common.persistence.rbac import PermissionGrant

__all__ = [
    "get_kvp_for_name",
    "get_values_for_names",
    "KeyValueLookup",
    "UserKeyValueLookup",
]

LOG = logging.getLogger(__name__)

def get_kvp_for_name(name):
    try:
        kvp_db = KeyValuePair.get_by_name(name)
    except ValueError:
        kvp_db = None

    return kvp_db

def get_values_for_names(names, default_value=None):
    """
    Retrieve values for the provided key names (multi get).

    If a KeyValuePair objects for a particular name doesn't exist, the dictionary will contain
    default_value for that name.

    :rtype: ``dict``
    """
    result = {}
    kvp_dbs = KeyValuePair.get_by_names(names=names)

    name_to_kvp_db_map = {}
    for kvp_db in kvp_dbs:
        name_to_kvp_db_map[kvp_db.name] = kvp_db.value

    for name in names:
        result[name] = name_to_kvp_db_map.get(name, default_value)

    return result

class BaseKeyValueLookup(object):

    scope = None
    _key_prefix = None

    def get_key_name(self):
        """
        Function which returns an original key name.

        :rtype: ``str``
        """
        key_name_parts = [DATASTORE_PARENT_SCOPE, self.scope]
        key_name = self._key_prefix.split(":", 1)

        if len(key_name) == 1:
            key_name = key_name[0]
        elif len(key_name) >= 2:
            key_name = key_name[1]
        else:
            key_name = ""

        key_name_parts.append(key_name)
        key_name = ".".join(key_name_parts)
        return key_name

class KeyValueLookup(BaseKeyValueLookup):

    scope = SYSTEM_SCOPE

    def __init__(
        self, prefix=None, key_prefix=None, cache=None, scope=FULL_SYSTEM_SCOPE
    ):
        if not scope:
            scope = FULL_SYSTEM_SCOPE

        if scope == SYSTEM_SCOPE:
            scope = FULL_SYSTEM_SCOPE

        self._prefix = prefix
        self._key_prefix = key_prefix or ""
        self._value_cache = cache or {}
        self._scope = scope

    def __str__(self):
        return self._value_cache[self._key_prefix]

    def __int__(self):
        return int(float(self))

    def __float__(self):
        return float(str(self))

    def __getitem__(self, key):
        return self._get(key)

    def __getattr__(self, name):
        return self._get(name)

    def _get(self, name):
        # get the value for this key and save in value_cache
        if self._key_prefix:
            key = "%s.%s" % (self._key_prefix, name)
        else:
            key = name

        if self._prefix:
            kvp_key = DATASTORE_KEY_SEPARATOR.join([self._prefix, key])
        else:
            kvp_key = key

        value = self._get_kv(kvp_key)
        self._value_cache[key] = value
        # return a KeyValueLookup as response since the lookup may not be complete e.g. if
        # the lookup is for 'key_base.key_value' it is likely that the calling code, e.g. Jinja,
        # will expect to do a dictionary style lookup for key_base and key_value as subsequent
        # calls. Saving the value in cache avoids extra DB calls.
        return KeyValueLookup(
            prefix=self._prefix,
            key_prefix=key,
            cache=self._value_cache,
            scope=self._scope,
        )

    def _get_kv(self, key):
        scope = self._scope
        LOG.debug("Lookup system kv: scope: %s and key: %s", scope, key)

        try:
            kvp = KeyValuePair.get_by_scope_and_name(scope=scope, name=key)
        except StackStormDBObjectNotFoundError:
            kvp = None

        if kvp:
            LOG.debug("Got value %s from datastore.", kvp.value)
        return kvp.value if kvp else ""

class UserKeyValueLookup(BaseKeyValueLookup):

    scope = USER_SCOPE

    def __init__(
        self, user, prefix=None, key_prefix=None, cache=None, scope=FULL_USER_SCOPE
    ):
        if not scope:
            scope = FULL_USER_SCOPE

        if scope == USER_SCOPE:
            scope = FULL_USER_SCOPE

        self._prefix = prefix
        self._key_prefix = key_prefix or ""
        self._value_cache = cache or {}
        self._user = user
        self._scope = scope

    def __str__(self):
        return self._value_cache[self._key_prefix]

    def __getitem__(self, key):
        return self._get(key)

    def __getattr__(self, name):
        return self._get(name)

    def _get(self, name):
        # get the value for this key and save in value_cache
        if self._key_prefix:
            key = "%s.%s" % (self._key_prefix, name)
        else:
            key = UserKeyReference(name=name, user=self._user).ref

        if self._prefix:
            kvp_key = DATASTORE_KEY_SEPARATOR.join([self._prefix, key])
        else:
            kvp_key = key

        value = self._get_kv(kvp_key)
        self._value_cache[key] = value
        # return a KeyValueLookup as response since the lookup may not be complete e.g. if
        # the lookup is for 'key_base.key_value' it is likely that the calling code, e.g. Jinja,
        # will expect to do a dictionary style lookup for key_base and key_value as subsequent
        # calls. Saving the value in cache avoids extra DB calls.
        return UserKeyValueLookup(
            prefix=self._prefix,
            user=self._user,
            key_prefix=key,
            cache=self._value_cache,
            scope=self._scope,
        )

    def _get_kv(self, key):
        scope = self._scope

        try:
            kvp = KeyValuePair.get_by_scope_and_name(scope=scope, name=key)
        except StackStormDBObjectNotFoundError:
            kvp = None

        return kvp.value if kvp else ""

def get_key_reference(scope, name, user=None):
    """
    Given a key name and user this method returns a new name (string ref)
    to address the key value pair in the context of that user.

    :param user: User to whom key belongs.
    :type user: ``str``

    :param name: Original name of the key.
    :type name: ``str``

    :rtype: ``str``
    """
    if scope == SYSTEM_SCOPE or scope == FULL_SYSTEM_SCOPE:
        return name
    elif scope == USER_SCOPE or scope == FULL_USER_SCOPE:
        if not user:
            raise InvalidUserException(
                "A valid user must be specified for user key ref."
            )
        return UserKeyReference(name=name, user=user).ref
    else:
        raise InvalidScopeException(
            'Scope "%s" is not valid. Allowed scopes are %s.' % (scope, ALLOWED_SCOPES)
        )

def get_uids(user):
    role_names = UserRoleAssignment.query(user=user).only("role").scalar("role")
    permission_grant_ids = Role.query(name__in=role_names).scalar("permission_grants")
    permission_grant_ids = sum(permission_grant_ids, [])
    permission_grants_filters = {}
    permission_grants_filters["id__in"] = permission_grant_ids
    uid = PermissionGrant.query(id__in=permission_grant_ids).scalar("resource_uid")
    return uid

def get_all_system_kvp_names_for_user(user):
    """
    Retrieve all the permission grants for a particular user.
    The result will return the key list

    :rtype: ``list``
    """

    uid = get_uids(user)
    key_list = []
    for u in uid:
        key_name = u.split(":")
        if len(key_name[2]) > 0 and key_name[2] not in key_list:
            key_list.append(key_name[2])

    return key_list
<commit_msg>Modified method to return only keyvaluepair keys<commit_after># Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

from st2common import log as logging

from st2common.constants.keyvalue import DATASTORE_PARENT_SCOPE
from st2common.constants.keyvalue import SYSTEM_SCOPE, FULL_SYSTEM_SCOPE
from st2common.constants.keyvalue import USER_SCOPE, FULL_USER_SCOPE
from st2common.constants.keyvalue import ALLOWED_SCOPES
from st2common.constants.keyvalue import DATASTORE_KEY_SEPARATOR
from st2common.exceptions.db import StackStormDBObjectNotFoundError
from st2common.exceptions.keyvalue import InvalidScopeException, InvalidUserException
from st2common.models.system.keyvalue import UserKeyReference
from st2common.persistence.keyvalue import KeyValuePair
from st2common.persistence.rbac import UserRoleAssignment
from st2common.persistence.rbac import Role
from st2common.persistence.rbac import PermissionGrant
from st2common.constants.types import ResourceType

__all__ = [
    "get_kvp_for_name",
    "get_values_for_names",
    "KeyValueLookup",
    "UserKeyValueLookup",
]

LOG = logging.getLogger(__name__)

def get_kvp_for_name(name):
    try:
        kvp_db = KeyValuePair.get_by_name(name)
    except ValueError:
        kvp_db = None

    return kvp_db

def get_values_for_names(names, default_value=None):
    """
    Retrieve values for the provided key names (multi get).

    If a KeyValuePair objects for a particular name doesn't exist, the dictionary will contain
    default_value for that name.

    :rtype: ``dict``
    """
    result = {}
    kvp_dbs = KeyValuePair.get_by_names(names=names)

    name_to_kvp_db_map = {}
    for kvp_db in kvp_dbs:
        name_to_kvp_db_map[kvp_db.name] = kvp_db.value

    for name in names:
        result[name] = name_to_kvp_db_map.get(name, default_value)

    return result

class BaseKeyValueLookup(object):

    scope = None
    _key_prefix = None

    def get_key_name(self):
        """
        Function which returns an original key name.

        :rtype: ``str``
        """
        key_name_parts = [DATASTORE_PARENT_SCOPE, self.scope]
        key_name = self._key_prefix.split(":", 1)

        if len(key_name) == 1:
            key_name = key_name[0]
        elif len(key_name) >= 2:
            key_name = key_name[1]
        else:
            key_name = ""

        key_name_parts.append(key_name)
        key_name = ".".join(key_name_parts)
        return key_name

class KeyValueLookup(BaseKeyValueLookup):

    scope = SYSTEM_SCOPE

    def __init__(
        self, prefix=None, key_prefix=None, cache=None, scope=FULL_SYSTEM_SCOPE
    ):
        if not scope:
            scope = FULL_SYSTEM_SCOPE

        if scope == SYSTEM_SCOPE:
            scope = FULL_SYSTEM_SCOPE

        self._prefix = prefix
        self._key_prefix = key_prefix or ""
        self._value_cache = cache or {}
        self._scope = scope

    def __str__(self):
        return self._value_cache[self._key_prefix]

    def __int__(self):
        return int(float(self))

    def __float__(self):
        return float(str(self))

    def __getitem__(self, key):
        return self._get(key)

    def __getattr__(self, name):
        return self._get(name)

    def _get(self, name):
        # get the value for this key and save in value_cache
        if self._key_prefix:
            key = "%s.%s" % (self._key_prefix, name)
        else:
            key = name

        if self._prefix:
            kvp_key = DATASTORE_KEY_SEPARATOR.join([self._prefix, key])
        else:
            kvp_key = key

        value = self._get_kv(kvp_key)
        self._value_cache[key] = value
        # return a KeyValueLookup as response since the lookup may not be complete e.g. if
        # the lookup is for 'key_base.key_value' it is likely that the calling code, e.g. Jinja,
        # will expect to do a dictionary style lookup for key_base and key_value as subsequent
        # calls. Saving the value in cache avoids extra DB calls.
        return KeyValueLookup(
            prefix=self._prefix,
            key_prefix=key,
            cache=self._value_cache,
            scope=self._scope,
        )

    def _get_kv(self, key):
        scope = self._scope
        LOG.debug("Lookup system kv: scope: %s and key: %s", scope, key)

        try:
            kvp = KeyValuePair.get_by_scope_and_name(scope=scope, name=key)
        except StackStormDBObjectNotFoundError:
            kvp = None

        if kvp:
            LOG.debug("Got value %s from datastore.", kvp.value)
        return kvp.value if kvp else ""

class UserKeyValueLookup(BaseKeyValueLookup):

    scope = USER_SCOPE

    def __init__(
        self, user, prefix=None, key_prefix=None, cache=None, scope=FULL_USER_SCOPE
    ):
        if not scope:
            scope = FULL_USER_SCOPE

        if scope == USER_SCOPE:
            scope = FULL_USER_SCOPE

        self._prefix = prefix
        self._key_prefix = key_prefix or ""
        self._value_cache = cache or {}
        self._user = user
        self._scope = scope

    def __str__(self):
        return self._value_cache[self._key_prefix]

    def __getitem__(self, key):
        return self._get(key)

    def __getattr__(self, name):
        return self._get(name)

    def _get(self, name):
        # get the value for this key and save in value_cache
        if self._key_prefix:
            key = "%s.%s" % (self._key_prefix, name)
        else:
            key = UserKeyReference(name=name, user=self._user).ref

        if self._prefix:
            kvp_key = DATASTORE_KEY_SEPARATOR.join([self._prefix, key])
        else:
            kvp_key = key

        value = self._get_kv(kvp_key)
        self._value_cache[key] = value
        # return a KeyValueLookup as response since the lookup may not be complete e.g. if
        # the lookup is for 'key_base.key_value' it is likely that the calling code, e.g. Jinja,
        # will expect to do a dictionary style lookup for key_base and key_value as subsequent
        # calls. Saving the value in cache avoids extra DB calls.
        return UserKeyValueLookup(
            prefix=self._prefix,
            user=self._user,
            key_prefix=key,
            cache=self._value_cache,
            scope=self._scope,
        )

    def _get_kv(self, key):
        scope = self._scope

        try:
            kvp = KeyValuePair.get_by_scope_and_name(scope=scope, name=key)
        except StackStormDBObjectNotFoundError:
            kvp = None

        return kvp.value if kvp else ""

def get_key_reference(scope, name, user=None):
    """
    Given a key name and user this method returns a new name (string ref)
    to address the key value pair in the context of that user.

    :param user: User to whom key belongs.
    :type user: ``str``

    :param name: Original name of the key.
    :type name: ``str``

    :rtype: ``str``
    """
    if scope == SYSTEM_SCOPE or scope == FULL_SYSTEM_SCOPE:
        return name
    elif scope == USER_SCOPE or scope == FULL_USER_SCOPE:
        if not user:
            raise InvalidUserException(
                "A valid user must be specified for user key ref."
            )
        return UserKeyReference(name=name, user=user).ref
    else:
        raise InvalidScopeException(
            'Scope "%s" is not valid. Allowed scopes are %s.' % (scope, ALLOWED_SCOPES)
        )

def get_uids(user):
    role_names = UserRoleAssignment.query(user=user).only("role").scalar("role")
    permission_grant_ids = Role.query(name__in=role_names).scalar("permission_grants")
    permission_grant_ids = sum(permission_grant_ids, [])
    permission_grants_filters = {}
    permission_grants_filters["id__in"] = permission_grant_ids
    permission_grants_filters["resource_type"] = ResourceType.KEY_VALUE_PAIR
    uid = PermissionGrant.query(id__in=permission_grant_ids, resource_type=ResourceType.KEY_VALUE_PAIR).scalar("resource_uid")
    return uid

def get_all_system_kvp_names_for_user(user):
    """
    Retrieve all the permission grants for a particular user.
    The result will return the key list

    :rtype: ``list``
    """

    uid = get_uids(user)
    key_list = []
    for u in uid:
        key_name = u.split(":")
        if len(key_name[2]) > 0 and key_name[2] not in key_list:
            key_list.append(key_name[2])

    return key_list
<|endoftext|>


<commit_before>"""
Alignment module.
"""
from __future__ import print_function
from array import array
import sys

# flags for global alignment

# The interpretation of the first flag is:
# An initial portion of seq1 may be skipped at no cost.
# This is equivalent to saying that in the alignment,
# gaps in the beginning of seq2 are free.
#
# The other flags have a similar meaning.
START_WITHIN_SEQ1 = 1
START_WITHIN_SEQ2 = 2
STOP_WITHIN_SEQ1 = 4
STOP_WITHIN_SEQ2 = 8

# Use this to get regular semiglobal alignment
# (all gaps in the beginning or end are free)
SEMIGLOBAL = START_WITHIN_SEQ1 | START_WITHIN_SEQ2 | STOP_WITHIN_SEQ1 | STOP_WITHIN_SEQ2

SCORE_MATCH = 1
SCORE_MISMATCH = -1
SCORE_DELETION = -2
SCORE_INSERTION = SCORE_DELETION

def _ansired(s):
	return "\x1b[1;31m" + s + "\x1b[00m"

# Pure Python implementation, fallback for when the C module is not available.
# Also useful for testing.
def pysemiglobalalign(s1, s2, print_table=False):
	"""
	Compute a semiglobal alignment of strings s1 and s2.

	Return a tuple (r1, r2, start1, stop1, start2, stop2, errors)
	where r1 and r2 are sequences of the same length containing the alignment
	(an INDEL is marked by '-').

	start1 is the position within r1 at which the part of s1, that is aligned, starts.
	start2 is the position within r1 at which the part of s1, that is aligned, ends.
	The same holds for start2, stop2.

	It is always the case that at least one of start1 and start2 is zero.

	It is always the case that either stop1==len(r1) or stop2==len(r2) or both
	(note that len(r1)==len(r2)). This is a property of semiglobal alignments.

	errors is the number of errors in the alignment.

	For example, pysemiglobalalign("SISSI", "MISSISSIPPI") returns:

	r1 = [ '-', '-', '-', 'S', 'I', 'S', 'S', 'I', '-', '-', '-']
	r2 = [ 'M', 'I', 'S', 'S', 'I', 'S', 'S', 'I', 'P', 'P', 'I']
	start1, stop1 = 0, 5
	start2, stop2 = 3, 8
	errors = 0

	This corresponds to the following alignment (formatted with print_alignment):
	SISSI
	|||||
	MISSISSIPPI
	"""

	#             s2 (n, j)
	#      --------------->
	#     |
	#  s1 | (m, i)
	#     |
	#     V
	m = len(s1)
	n = len(s2)

	# the DP and backtrace table are both stored column-wise
	# It's much faster to use two tables instead
	# of one with tuples. (Tried both.)
	columns = [ array('h', (m+1)*[0]) for x in xrange(n+1) ]
	backtrace = [ array('h', (m+1)*[0]) for x in xrange(n+1) ]

	# direction constants for backtrace table
	LEFT = 1
	TOP = 2
	DIAG = 3

	# fill DP table (using unit costs)
	# outer loop goes over columns
	for j in range(1, n+1):
		cur_column = columns[j]
		prev_column = columns[j-1]
		for i in range(1, m+1):
			bt = DIAG
			score = prev_column[i-1] + (SCORE_MATCH if (s1[i-1] == s2[j-1]) else SCORE_MISMATCH)
			tmp = cur_column[i-1] + SCORE_INSERTION
			if tmp > score:
				bt = LEFT
				score = tmp
			tmp = prev_column[i] + SCORE_DELETION
			if tmp > score:
				bt = TOP
				score = tmp
			cur_column[i] = score
			backtrace[j][i] = bt

	# find position with highest score in last column or last row
	last_row = [ (columns[j][m], j) for j in xrange(n+1) ]
	row_best, best_j = max(last_row)

	last_column = [ (columns[n][i], i) for i in xrange(m+1) ]
	col_best, best_i = max(last_column)

	if row_best > col_best:
		best_i = m
		best = row_best
	else:
		best_j = n
		best = col_best

	if print_table:
		print("s1:", s1)
		print("s2:", s2)
		print("best i,j", best_i, best_j)
		print("      ", end="")
		for j in xrange(n):
			print("%3c" % s2[j], end="")
		print()
		for i in xrange(m+1):
			if i > 0:
				print("%3c" % s1[i-1], end="")
			else:
				print("   ", end="")
			for j in xrange(n+1):
				if (i,j) == (best_i, best_j):
					st = _ansired("%3d")
				else:
					st = "%3d"
				sys.stdout.write(st % columns[j][i])
			print()
		print()

	# now trace back (from "lower right" corner)
	r1 = []
	r2 = []
	i = m
	j = n

	# first, walk from the lower right corner to the
	# position where we found the maximum score
	if i == best_i: # we are in the last row
		while j > best_j:
			r1.append('-')
			r2.append(s2[j-1])
			j -= 1
	else: # we are in the last column
		while i > best_i:
			r1.append(s1[i-1])
			r2.append('-')
			i -= 1

	rlen = len(r1)

	errors = 0
	# now, the actual backtracing.
	# we build reverse sequences while backtracing and
	# reverse them afterwards.
	while i > 0 and j > 0: #columns[j][i] > 0:
		score = columns[j][i]
		direction = backtrace[j][i]
		if direction == DIAG:
			if s1[i-1] != s2[j-1]:
				errors += 1
			r1.append(s1[i-1])
			r2.append(s2[j-1])
			i -= 1
			j -= 1
		elif direction == TOP:
			r1.append('-')
			r2.append(s2[j-1])
			errors += 1
			j -= 1
		elif direction == LEFT:
			r1.append(s1[i-1])
			r2.append('-')
			errors += 1
			i -= 1

	# compute the length of the actual alignment (ignoring ends)
	length = len(r1) - rlen

	start1 = i
	start2 = j

	# continue from where we are to the upper left corner
	while j > 0:
		r1.append('-')
		r2.append(s2[j-1])
		j -= 1
	while i > 0:
		r1.append(s1[i-1])
		r2.append('-')
		i -= 1
	assert i == 0 and j == 0

	#TODO assert columns[best_j][best_i] == length - 2*errors
	r1.reverse()
	r2.reverse()

	stop1 = best_i
	stop2 = best_j
	return (r1, r2, start1, stop1, start2, stop2, errors)

def semiglobalalign(s1, s2):
	"""
	Computes an end-gap free alignment (also called free-shift alignment) of
	strings s1 and s2.
	"""
	return globalalign(s1, s2, SEMIGLOBAL)

#try:
from cutadapt.calign import globalalign
#except ImportError:
	#print("Warning: C implementation of alignment algorithm not available. Using slow Python version!", file=sys.stderr)
	#semiglobalalign = pysemiglobalalign
<commit_msg>import the new globalalign_locate function; remove (unused) wrapper function semiglobalalign<commit_after>"""
Alignment module.
"""
from __future__ import print_function
from array import array
import sys

# flags for global alignment

# The interpretation of the first flag is:
# An initial portion of seq1 may be skipped at no cost.
# This is equivalent to saying that in the alignment,
# gaps in the beginning of seq2 are free.
#
# The other flags have an equivalent meaning.
START_WITHIN_SEQ1 = 1
START_WITHIN_SEQ2 = 2
STOP_WITHIN_SEQ1 = 4
STOP_WITHIN_SEQ2 = 8

# Use this to get regular semiglobal alignment
# (all gaps in the beginning or end are free)
SEMIGLOBAL = START_WITHIN_SEQ1 | START_WITHIN_SEQ2 | STOP_WITHIN_SEQ1 | STOP_WITHIN_SEQ2

SCORE_MATCH = 1
SCORE_MISMATCH = -1
SCORE_DELETION = -2
SCORE_INSERTION = SCORE_DELETION

def _ansired(s):
	return "\x1b[1;31m" + s + "\x1b[00m"

# Pure Python implementation, fallback for when the C module is not available.
# Also useful for testing.
def pysemiglobalalign(s1, s2, print_table=False):
	"""
	Compute a semiglobal alignment of strings s1 and s2.

	Return a tuple (r1, r2, start1, stop1, start2, stop2, errors)
	where r1 and r2 are sequences of the same length containing the alignment
	(an INDEL is marked by '-').

	start1 is the position within r1 at which the part of s1, that is aligned, starts.
	stop2 is the position within r1 at which the part of s1, that is aligned, ends.
	The same holds for start2, stop2.

	It is always the case that at least one of start1 and start2 is zero.

	It is always the case that either stop1==len(r1) or stop2==len(r2) or both
	(note that len(r1)==len(r2)). This is a property of semiglobal alignments.

	errors is the number of errors in the alignment.

	For example, pysemiglobalalign("SISSI", "MISSISSIPPI") returns:

	r1 = [ '-', '-', '-', 'S', 'I', 'S', 'S', 'I', '-', '-', '-']
	r2 = [ 'M', 'I', 'S', 'S', 'I', 'S', 'S', 'I', 'P', 'P', 'I']
	start1, stop1 = 0, 5
	start2, stop2 = 3, 8
	errors = 0

	This corresponds to the following alignment (formatted with print_alignment):
	SISSI
	|||||
	MISSISSIPPI
	"""

	#             s2 (n, j)
	#      --------------->
	#     |
	#  s1 | (m, i)
	#     |
	#     V
	m = len(s1)
	n = len(s2)

	# the DP and backtrace table are both stored column-wise
	# It's much faster to use two tables instead
	# of one with tuples. (Tried both.)
	columns = [ array('h', (m+1)*[0]) for x in xrange(n+1) ]
	backtrace = [ array('h', (m+1)*[0]) for x in xrange(n+1) ]

	# direction constants for backtrace table
	LEFT = 1
	TOP = 2
	DIAG = 3

	# fill DP table (using unit costs)
	# outer loop goes over columns
	for j in range(1, n+1):
		cur_column = columns[j]
		prev_column = columns[j-1]
		for i in range(1, m+1):
			bt = DIAG
			score = prev_column[i-1] + (SCORE_MATCH if (s1[i-1] == s2[j-1]) else SCORE_MISMATCH)
			tmp = cur_column[i-1] + SCORE_INSERTION
			if tmp > score:
				bt = LEFT
				score = tmp
			tmp = prev_column[i] + SCORE_DELETION
			if tmp > score:
				bt = TOP
				score = tmp
			cur_column[i] = score
			backtrace[j][i] = bt

	# find position with highest score in last column or last row
	last_row = [ (columns[j][m], j) for j in xrange(n+1) ]
	row_best, best_j = max(last_row)

	last_column = [ (columns[n][i], i) for i in xrange(m+1) ]
	col_best, best_i = max(last_column)

	if row_best > col_best:
		best_i = m
		best = row_best
	else:
		best_j = n
		best = col_best

	if print_table:
		print("s1:", s1)
		print("s2:", s2)
		print("best i,j", best_i, best_j)
		print("      ", end="")
		for j in xrange(n):
			print("%3c" % s2[j], end="")
		print()
		for i in xrange(m+1):
			if i > 0:
				print("%3c" % s1[i-1], end="")
			else:
				print("   ", end="")
			for j in xrange(n+1):
				if (i,j) == (best_i, best_j):
					st = _ansired("%3d")
				else:
					st = "%3d"
				sys.stdout.write(st % columns[j][i])
			print()
		print()

	# now trace back (from "lower right" corner)
	r1 = []
	r2 = []
	i = m
	j = n

	# first, walk from the lower right corner to the
	# position where we found the maximum score
	if i == best_i: # we are in the last row
		while j > best_j:
			r1.append('-')
			r2.append(s2[j-1])
			j -= 1
	else: # we are in the last column
		while i > best_i:
			r1.append(s1[i-1])
			r2.append('-')
			i -= 1

	rlen = len(r1)

	errors = 0
	# now, the actual backtracing.
	# we build reverse sequences while backtracing and
	# reverse them afterwards.
	while i > 0 and j > 0: #columns[j][i] > 0:
		score = columns[j][i]
		direction = backtrace[j][i]
		if direction == DIAG:
			if s1[i-1] != s2[j-1]:
				errors += 1
			r1.append(s1[i-1])
			r2.append(s2[j-1])
			i -= 1
			j -= 1
		elif direction == TOP:
			r1.append('-')
			r2.append(s2[j-1])
			errors += 1
			j -= 1
		elif direction == LEFT:
			r1.append(s1[i-1])
			r2.append('-')
			errors += 1
			i -= 1

	# compute the length of the actual alignment (ignoring ends)
	length = len(r1) - rlen

	start1 = i
	start2 = j

	# continue from where we are to the upper left corner
	while j > 0:
		r1.append('-')
		r2.append(s2[j-1])
		j -= 1
	while i > 0:
		r1.append(s1[i-1])
		r2.append('-')
		i -= 1
	assert i == 0 and j == 0

	#TODO assert columns[best_j][best_i] == length - 2*errors
	r1.reverse()
	r2.reverse()

	stop1 = best_i
	stop2 = best_j
	return (r1, r2, start1, stop1, start2, stop2, errors)

#try:
from cutadapt.calign import globalalign, globalalign_locate
#except ImportError:
	#print("Warning: C implementation of alignment algorithm not available. Using slow Python version!", file=sys.stderr)
	#semiglobalalign = pysemiglobalalign
<|endoftext|>


<commit_before>"""
"""

from crocs.yacc import Lexer, LexMap, LexNode, Rule, Grammar, Struct, Yacc
from crocs.token import Plus, Minus, LP, RP, Mul, Div, Num, Blank

class CalcTokens:
    lexmap = LexMap()
    LexNode(lexmap, r'\+', Plus)
    LexNode(lexmap, r'\-', Minus)
    LexNode(lexmap, r'\(', LP)
    LexNode(lexmap, r'\)', RP)
    LexNode(lexmap, r'\*', Mul)
    LexNode(lexmap, r'\/', Div)

    LexNode(lexmap, r'[0-9]+', Num)
    LexNode(lexmap, r' +', Blank)

class CalcGrammar:
    expression = Grammar(recursive=True)
    term       = Grammar()
    factor     = Grammar()

    r_plus = Rule(expression, Plus, term)
    r_minus = Rule(expression, Minus, term)
    expression.add(term, r_plus,  r_minus)

    r_mul = Rule(term, Mul, factor)
    r_div = Rule(term, Div, factor)
    term.add(factor, r_mul, r_div)

    r_paren = Rule(LP, expression, RP)
    r_num = Rule(Num)
    factor.add(r_paren, Num)

    expression.discard(Blank)

class CalcParser(Yacc):
    def __init__(self):
        self.lexer = Lexer(CalcTokens.lexmap)
        super(CalcParser, self).__init__(CalcGrammar.expression)

    def calc(self, data):
        self.lexer.feed(data)
        tokens = self.lexer.run()
        return self.build(tokens)

data = '1 + 2 * (3 /(4 - (5 - (6 + (7 + (8 + (9 + (10 + (11 + (12 + (13+(14 * 15 + 16))))))))))))'
parser = CalcParser()
ptree = parser.calc(data)
print('Consumed:', list(ptree))

<commit_msg>Adjusting example.<commit_after>"""
"""

from crocs.yacc import Lexer, LexMap, LexNode, Rule, Grammar, Struct, Yacc
from crocs.token import Plus, Minus, LP, RP, Mul, Div, Num, Blank

class CalcTokens:
    lexmap = LexMap()
    LexNode(lexmap, r'\+', Plus)
    LexNode(lexmap, r'\-', Minus)
    LexNode(lexmap, r'\(', LP)
    LexNode(lexmap, r'\)', RP)
    LexNode(lexmap, r'\*', Mul)
    LexNode(lexmap, r'\/', Div)

    LexNode(lexmap, r'[0-9]+', Num)
    LexNode(lexmap, r' +', Blank)

class CalcGrammar:
    expression = Grammar(recursive=True)
    term       = Grammar(recursive=True)
    factor     = Grammar()

    r_plus = Rule(expression, Plus, term)
    r_minus = Rule(expression, Minus, term)
    expression.add(term, r_plus,  r_minus)

    r_mul = Rule(term, Mul, factor)
    r_div = Rule(term, Div, factor)
    term.add(factor, r_mul, r_div)

    r_paren = Rule(LP, expression, RP)
    factor.add(r_paren, Num)

    expression.discard(Blank)

class CalcParser(Yacc):
    def __init__(self):
        self.lexer = Lexer(CalcTokens.lexmap)
        super(CalcParser, self).__init__(CalcGrammar.expression)

    def calc(self, data):
        self.lexer.feed(data)
        tokens = self.lexer.run()
        return self.build(tokens)

data = '1 + 2 * (3 /(4 - (5 - (6 + (7 + (8 + (9 + (10 + (11 + (12 + (13+(14 * 15 + 16))))))))))))'
parser = CalcParser()
ptree = parser.calc(data)
print('Consumed:', list(ptree))

<|endoftext|>


<commit_before>#!/usr/bin/python
import datetime
import time
import sys
import os
import cProfile
import cPickle
import collections
import logging
logging.basicConfig(level=logging.DEBUG, format='%(asctime)-15s %(levelname)s %(filename)s:%(funcName)s:%(lineno)s %(message)s')
import unittest
from Timeseries import Timeseries as Timeseries
from TimeseriesArray import TimeseriesArray as TimeseriesArray
from DataLogger import DataLogger as DataLogger
import DataLoggerHelper as dh

class Test(unittest.TestCase):
    """Unittest Classs for WsusDb"""

    basedir = "/var/rrd/"
    project = "test"
    tablename = "fcIfC3AccountingTable"
    datestring = "2015-05-23"
    ts_keyname= "ts"
    datalogger = DataLogger(basedir, project, tablename)
    start_ts, stop_ts = dh.get_ts_from_datestring(datestring)
    keys = ("hostname", "ifDescr")
    # dataset uncompressed
    result_23 = dh.read_day(datalogger, ts_keyname=ts_keyname, index_keynames=keys, datestring="2015-05-23")
    result_23a = datalogger.read_day(datestring="2015-05-23")
    # dataset compressed
    result_24 = dh.read_day(datalogger, ts_keyname=ts_keyname, index_keynames=keys, datestring="2015-05-24")
    result_24a = datalogger.read_day(datestring="2015-05-24")

    def no_test_public_methods(self):
        """
        Method to test if all public methods of testclass are tested
        """
        testclass = dh
        state = True
        for item in dir(testclass):
            method = getattr(testclass, item)
            if (callable(method)) and (not item.startswith("_")):
                test_method = "test_%s" % item
                if test_method not in dir(self):
                    logging.error("No Testmethod found for Method %s" % item)
                    state = False
        self.assertIs(state, True)

    def test_get_ts_from_datestring(self):
        datestring = "1970-01-01"
        start_ts, stop_ts = dh.get_ts_from_datestring(datestring)
        self.assertEqual(start_ts, time.timezone)
        self.assertEqual(stop_ts, 86399)

    def test_read_one(self):
        index_dict = collections.OrderedDict(sorted({"hostname":'fcb-sr1-8gb-12', "ifDescr":'fc1/5'}.items()))
        result = dh.read_one(self.datalogger, ts_keyname=self.ts_keyname, index_dict=index_dict, start_ts=self.start_ts, stop_ts=self.stop_ts)
        self.assertIs(type(result), TimeseriesArray)

    def test_read_day(self):
        keys = ("hostname", "ifDescr")
        self.assertIs(type(self.result_24), TimeseriesArray)

    def test_dump_list(self):
        index_dict = collections.OrderedDict(sorted({"hostname":'fcb-sr1-8gb-12', "ifDescr":'fc1/5'}.items()))
        result1 = dh.read_one(self.datalogger, ts_keyname=self.ts_keyname, index_dict=index_dict, start_ts=self.start_ts, stop_ts=self.stop_ts)
        result2 = result1[('fcb-sr1-8gb-12', 'fc1/5')].dump_list(('fcIfC3OutFrames',))
        self.assertIs(type(result2), list)
        self.assertIs(type(result2[0]), list)

    def test_dump_dict(self):
        index_dict = collections.OrderedDict(sorted({"hostname":'fcb-sr1-8gb-12', "ifDescr":'fc1/5'}.items()))
        result1 = dh.read_one(self.datalogger, ts_keyname=self.ts_keyname, index_dict=index_dict, start_ts=self.start_ts, stop_ts=self.stop_ts)
        result2 = result1[('fcb-sr1-8gb-12', 'fc1/5')].dump_dict(('fcIfC3OutFrames',))
        self.assertIs(type(result2), dict)
        key, value = result2.items()[0]
        self.assertIs(type(key), float)
        self.assertIs(type(value), dict)

    def test_dump_ts_dict(self):
        index_dict = collections.OrderedDict(sorted({"hostname":'fcb-sr1-8gb-12', "ifDescr":'fc1/5'}.items()))
        result1 = dh.read_one(self.datalogger, ts_keyname=self.ts_keyname, index_dict=index_dict, start_ts=self.start_ts, stop_ts=self.stop_ts)
        result2 = result1[('fcb-sr1-8gb-12', 'fc1/5')].dump_ts_dict(('fcIfC3OutFrames',))
        self.assertIs(type(result2), dict)
        key, value = result2.items()[0]
        self.assertIs(type(key), float)
        self.assertIs(type(value), tuple)

    def test_getitem(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        result1 = self.result_23[("fca-sr2-07bc", "bay13")]
        self.assertIs(type(result1), Timeseries)

    def test_get_stats(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        result1 = self.result_23[("fca-sr2-27bc", "bay2")].get_stats()
        should = {'std': 0.0, 'count': 288, 'min': 0.0, 'max': 0.0, 'sum': 0.0, 'median': 0.0, 'first': 0.0, 'geomean': 0.0, 'avg': 0.0, 'latest': 0.0}
        self.assertIs(type(result1), dict)
        self.assertTrue(result1['fcIfC3InOctets']["median"] == should["median"])

    def test_get_stat(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        result1 = self.result_23[("fca-sr2-27bc", "bay2")].get_stat(func='std')
        self.assertIs(type(result1), dict)
        should = {'index': 0.0, 'fcIfC3InFrames': 0.0, 'fcIfC3OutOctets': 0.0, 'fcIfC3InOctets': 0.0, 'fcIfC3OutFrames': 0.0, 'fcIfC3Discards': 0.0}
        self.assertTrue(result1 == should)

    def test_add_derive_col(self):
        keys = ("hostname", "ifDescr")
        result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        result.add_derive_col("fcIfC3InOctets", "fcIfC3InOctets_d")
        # the sum of the values of the derived series, has to be to
        # equal to last-first of the original series
        last = result[('fcb-sr1-8gb-12', 'fc1/5')].get_stat("last")
        first = result[('fcb-sr1-8gb-12', 'fc1/5')].get_stat("first")
        total_original = last["fcIfC3InOctets"] - first["fcIfC3InOctets"]
        total_derived = result[('fcb-sr1-8gb-12', 'fc1/5')].get_stat("sum")["fcIfC3InOctets_d"]
        #print total_original, total_derived
        self.assertTrue(total_original == total_derived)
        # so this is a test dataset, the result has to be fix
        self.assertEqual(total_original, 2422898425856.0)

    def test_per_s_col(self):
        keys = ("hostname", "ifDescr")
        result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        result.add_per_s_col("fcIfC3InOctets", "fcIfC3InOctets_s")
        print result.get_stats("fcIfC3InOctets_s")
        # the sum of the values of the derived series, has to be to
        # equal to last-first of the original series
        start_ts = result[('fcb-sr1-8gb-12', 'fc1/5')].get_start_ts()
        stop_ts = result[('fcb-sr1-8gb-12', 'fc1/5')].get_stop_ts()
        duration = stop_ts - start_ts
        last = result[('fcb-sr1-8gb-12', 'fc1/5')].get_stat("last")
        first = result[('fcb-sr1-8gb-12', 'fc1/5')].get_stat("first")
        total_original = last["fcIfC3InOctets"] - first["fcIfC3InOctets"]
        print "[('fcb-sr1-8gb-12', 'fc1/5')] fcIfC3InOctets : %s" % total_original
        total_per_s = result[('fcb-sr1-8gb-12', 'fc1/5')].get_stat("sum")["fcIfC3InOctets_s"] * duration
        print "[('fcb-sr1-8gb-12', 'fc1/5')] fcIfC3InOctets : %s" % total_per_s
        #print total_original, total_derived
        self.assertTrue(total_original == total_per_s)
        # so this is a test dataset, the result has to be fix
        self.assertEqual(total_original, 2422898425856.0)

    def test_slice(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        timeseries = self.result_23[('fcb-sr1-8gb-12', 'fc1/5')]
        subset = timeseries.slice(('fcIfC3OutOctets', 'fcIfC3InOctets'))
        self.assertTrue(('fcIfC3OutOctets', 'fcIfC3InOctets') == subset.get_headers())

    def test_serie(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        timeseries = self.result_23[('fcb-sr1-8gb-12', 'fc1/5')]
        serie = timeseries.get_serie('fcIfC3Discards')
        self.assertTrue(len(serie) == 288)
        self.assertTrue(type(serie) == tuple)
        self.assertTrue(serie[0] == 0.0)

    def test_times(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        timeseries = self.result_23[('fcb-sr1-8gb-12', 'fc1/5')]
        serie = timeseries.get_times()
        #print serie, type(serie), len(serie)
        self.assertTrue(len(serie) == 288)
        self.assertTrue(type(serie) == list)
        self.assertEqual(serie[0], 1432332032.0)
        self.assertEqual(serie[-1], 1432418176.0)

    def test_get_single_value(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        timeseries = self.result_23[('fcb-sr1-8gb-12', 'fc1/5')]
        #print "%f" % timeseries.get_single_value(1432386304.0, "fcIfC3OutOctets")
        self.assertTrue(timeseries.get_single_value(1432386304.0, "fcIfC3OutOctets") == 4245577319579648.000000)

    def no_test_save_csv(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        timeseries = self.result_23[('fcb-sr1-8gb-12', 'fc1/5')]
        headers = timeseries.get_headers()
        old_stats = timeseries.get_stats()
        timeseries.save_csv("/tmp/test.csv")
        timeseries.load_csv(headers, ts_keyname=self.ts_keyname, filename="/tmp/test.csv")
        new_stats = timeseries.get_stats()
        #print old_stats
        #print new_stats
        self.assertTrue(old_stats == new_stats)

    def no_test_save_csv(self):
        keys = ("hostname", "ifDescr")
        #result = dh.read_day(self.datalogger, ts_keyname=self.ts_keyname, index_keynames=keys, datestring="2015-05-23")
        dh.save_tsa(self.datalogger, self.result_23)
        print self.result_23.keys()
        timeseries = self.result_23[(u'fcb-sr1-8gb-12', u'fc1/5')]
        old_stats = timeseries.get_stats()
        loaded_tsa = dh.load_tsa(self.datalogger, self.ts_keyname, keys, 1432339206, 1432425382)
        loaded_timeseries = loaded_tsa[(u'fcb-sr1-8gb-12', u'fc1/5')]
        loaded_stats = loaded_timeseries.get_stats()
        #print old_stats
        #print loaded_stats
        self.assertTrue(old_stats == loaded_stats)

    def test_get_group_by_tsa(self):
        grouped_tsa = self.result_23.get_group_by_tsa(("hostname",), lambda a: sum(a))
        print grouped_tsa.keys()
        print grouped_tsa[('fcb-sr1-8gb-12', )].get_stats()
        self.assertEqual(grouped_tsa[('fcb-sr1-8gb-12', )].get_stats()["index"]['sum'], 329895641088.0)

    def test_interval(self):
        """
        tests TimeseriesArray method sanitize
        indirect test of Timeseries get_interval method also
        """
        self.result_23.sanitize()

    def test_get_scatter_data(self):
        tsa = self.result_23a.slice([u'fcIfC3InFrames', u'fcIfC3OutFrames'])
        tsa.add_per_s_col(u'fcIfC3InFrames', u'fcIfC3InFrames_s')
        tsa.add_per_s_col(u'fcIfC3OutFrames', u'fcIfC3OutFrames_s')
        tsa.remove_col(u'fcIfC3InFrames')
        tsa.remove_col(u'fcIfC3OutFrames')
        tsa_grouped = tsa.get_group_by_tsa(("hostname",), lambda a: sum(a))
        data = self.datalogger.get_scatter_data(tsa_grouped, u'fcIfC3InFrames_s', u'fcIfC3OutFrames_s', "mean")
        self.assertEqual(len(data), 22)
        self.assertEqual(data[0]["name"], "fca-sr1-19bc")
        self.assertEqual(data[0]["data"][0][0], 43726)
        self.assertEqual(data[0]["data"][0][1], 25608)

def test():
    logging.basicConfig(level=logging.DEBUG)

if __name__ == "__main__":
    unittest.main()
    #main()
    #cProfile.run("main()")
<commit_msg>Delete Test.py<commit_after><|endoftext|>


<commit_before>##
#    Copyright (C) 2013 Jessica Tallon & Matt Molyneaux
#
#    This file is part of Inboxen.
#
#    Inboxen is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    Inboxen is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with Inboxen.  If not, see <http://www.gnu.org/licenses/>.
##

from datetime import datetime
import random

from django.utils.translation import ugettext as _
from django import forms
from django.contrib import messages

from pytz import utc
import watson

from inboxen import models
from queue.delete import tasks

__all__ = ["InboxAddForm", "InboxEditForm", "InboxRestoreForm"]

class InboxAddForm(forms.ModelForm):
    exclude_from_unified = forms.BooleanField(required=False, label=_("Exclude from Unified Inbox"))

    def __init__(self, request, initial=None, *args, **kwargs):
        self.request = request # needed to create the inbox

        if not initial:
            initial = {
                "inbox": None, # This is filled in by the manager.create
                "domain": random.choice(models.Domain.objects.all()),
            }

        super(InboxAddForm, self).__init__(initial=initial, *args, **kwargs)
        # Remove empty option "-------"
        self.fields["domain"].empty_label = None

    class Meta:
        model = models.Inbox
        fields = ["domain", "tags"]
        widgets = {
            "tags": forms.TextInput(attrs={'placeholder': 'Tag1, Tag2, ...'})
            }

    def save(self, commit=True):
        # We're ignoring commit, should we?
        # We want this instance created by .create() so we will ignore self.instance
        # which is created just by model(**data)
        data = self.cleaned_data.copy()
        tags = data.pop("tags")
        excludes = data.pop("exclude_from_unified", False)

        self.instance = self.request.user.inbox_set.create(**data)
        self.instance.tags = tags
        self.instance.flags.exclude_from_unified = excludes
        self.instance.save()

        messages.success(self.request, _("{0}@{1} has been created.").format(self.instance.inbox, self.instance.domain.domain))
        return self.instance

class InboxSecondaryEditForm(forms.Form):
    """A subform to hold dangerous operations that will result in loss
    of data.
    """
    clear_inbox = forms.BooleanField(required=False, label=_("Clear Inbox"), help_text=_("Delete all emails in this Inbox. Cannot be undone."))
    disable_inbox = forms.BooleanField(required=False, label=_("Disable Inbox"), help_text=_("This Inbox will no longer receive emails."))

    def __init__(self, instance, *args, **kwargs):
        super(InboxSecondaryEditForm, self).__init__(*args, **kwargs)
        self.fields["disable_inbox"].initial = bool(instance.flags.disabled)

class InboxEditForm(forms.ModelForm):
    exclude_from_unified = forms.BooleanField(required=False, label=_("Exclude from Unified Inbox"))

    class Meta:
        model = models.Inbox
        fields = ["tags"]
        widgets = {
            "tags": forms.TextInput(attrs={'placeholder': 'Tag1, Tag2, ...'})
            }

    def __init__(self, request, initial=None, instance=None, *args, **kwargs):
        self.request = request
        super(InboxEditForm, self).__init__(instance=instance, initial=initial, *args, **kwargs)
        self.fields["exclude_from_unified"].initial = bool(self.instance.flags.exclude_from_unified)
        self.subform = InboxSecondaryEditForm(instance=self.instance, **kwargs)

    def save(self, commit=True):
        data = self.cleaned_data.copy()
        if self.subform.is_valid():
            data.update(self.subform.cleaned_data.copy())

        self.instance.flags.exclude_from_unified = data.pop("exclude_from_unified", False)
        self.instance.flags.disabled = data.pop("disable_inbox", False)
        clear_inbox = data.pop("clear_inbox", False)

        if not commit:
            return self.instance

        if clear_inbox:
            tasks.batch_delete_items.delay("email", kwargs={'inbox__id': self.instance.id})
            messages.warning(self.request, _("All emails in {0}@{1} are being deleted.").format(self.instance.inbox, self.instance.domain.domain))

        self.instance.save()

        return self.instance

class InboxRestoreForm(InboxEditForm):
    def save(self, commit=True):
        self.instance.flags.deleted = False
        self.instance.created = datetime.now(utc)
        return super(InboxRestoreForm, self).save(commit)
<commit_msg>Mark emails as deleted before they are actually deleted<commit_after>##
#    Copyright (C) 2013 Jessica Tallon & Matt Molyneaux
#
#    This file is part of Inboxen.
#
#    Inboxen is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    Inboxen is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with Inboxen.  If not, see <http://www.gnu.org/licenses/>.
##

from datetime import datetime
import random

from django.utils.translation import ugettext as _
from django import forms
from django.contrib import messages

from pytz import utc
import watson

from inboxen import models
from queue.delete import tasks

__all__ = ["InboxAddForm", "InboxEditForm", "InboxRestoreForm"]

class InboxAddForm(forms.ModelForm):
    exclude_from_unified = forms.BooleanField(required=False, label=_("Exclude from Unified Inbox"))

    def __init__(self, request, initial=None, *args, **kwargs):
        self.request = request # needed to create the inbox

        if not initial:
            initial = {
                "inbox": None, # This is filled in by the manager.create
                "domain": random.choice(models.Domain.objects.all()),
            }

        super(InboxAddForm, self).__init__(initial=initial, *args, **kwargs)
        # Remove empty option "-------"
        self.fields["domain"].empty_label = None

    class Meta:
        model = models.Inbox
        fields = ["domain", "tags"]
        widgets = {
            "tags": forms.TextInput(attrs={'placeholder': 'Tag1, Tag2, ...'})
            }

    def save(self, commit=True):
        # We're ignoring commit, should we?
        # We want this instance created by .create() so we will ignore self.instance
        # which is created just by model(**data)
        data = self.cleaned_data.copy()
        tags = data.pop("tags")
        excludes = data.pop("exclude_from_unified", False)

        self.instance = self.request.user.inbox_set.create(**data)
        self.instance.tags = tags
        self.instance.flags.exclude_from_unified = excludes
        self.instance.save()

        messages.success(self.request, _("{0}@{1} has been created.").format(self.instance.inbox, self.instance.domain.domain))
        return self.instance

class InboxSecondaryEditForm(forms.Form):
    """A subform to hold dangerous operations that will result in loss
    of data.
    """
    clear_inbox = forms.BooleanField(required=False, label=_("Clear Inbox"), help_text=_("Delete all emails in this Inbox. Cannot be undone."))
    disable_inbox = forms.BooleanField(required=False, label=_("Disable Inbox"), help_text=_("This Inbox will no longer receive emails."))

    def __init__(self, instance, *args, **kwargs):
        super(InboxSecondaryEditForm, self).__init__(*args, **kwargs)
        self.fields["disable_inbox"].initial = bool(instance.flags.disabled)

class InboxEditForm(forms.ModelForm):
    exclude_from_unified = forms.BooleanField(required=False, label=_("Exclude from Unified Inbox"))

    class Meta:
        model = models.Inbox
        fields = ["tags"]
        widgets = {
            "tags": forms.TextInput(attrs={'placeholder': 'Tag1, Tag2, ...'})
            }

    def __init__(self, request, initial=None, instance=None, *args, **kwargs):
        self.request = request
        super(InboxEditForm, self).__init__(instance=instance, initial=initial, *args, **kwargs)
        self.fields["exclude_from_unified"].initial = bool(self.instance.flags.exclude_from_unified)
        self.subform = InboxSecondaryEditForm(instance=self.instance, **kwargs)

    def save(self, commit=True):
        data = self.cleaned_data.copy()
        if self.subform.is_valid():
            data.update(self.subform.cleaned_data.copy())

        self.instance.flags.exclude_from_unified = data.pop("exclude_from_unified", False)
        self.instance.flags.disabled = data.pop("disable_inbox", False)
        clear_inbox = data.pop("clear_inbox", False)

        if not commit:
            return self.instance

        if clear_inbox:
            emails = self.instance.email_set.all()
            emails.update(flags=F('flags').bitor(self.model.flags.deleted))
            tasks.batch_delete_items.delay("email", kwargs={'inbox__id': self.instance.id})
            messages.warning(self.request, _("All emails in {0}@{1} are being deleted.").format(self.instance.inbox, self.instance.domain.domain))

        self.instance.save()

        return self.instance

class InboxRestoreForm(InboxEditForm):
    def save(self, commit=True):
        self.instance.flags.deleted = False
        self.instance.created = datetime.now(utc)
        return super(InboxRestoreForm, self).save(commit)
<|endoftext|>


<commit_before>"""
The democracy API.

we have some public resources which you can get annonymousy
but for the private data we require oauth authentication

"""
import datetime

from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.core.paginator import Paginator
from django.core.paginator import InvalidPage
from django.core.paginator import EmptyPage
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.contenttypes.models import ContentType

from piston.handler import BaseHandler, AnonymousBaseHandler
from piston.utils import rc
from piston.utils import validate

from democracy.api.forms import IssueForm
from democracy.api.forms import VoteForm
from democracy.api.forms import MultiplyForm
from democracy.issue.models import Issue
from democracy.voting.models import Vote
from democracy.profiles.models import UserProfile
from democracy.gamelogic.models import MultiplyIssue

import gamelogic.actions

from piston.models import Token

def paginate(request, qs):
    paginator = Paginator(qs, 3)  # TODO: add to settings.py
    try :
        pageno = int(request.GET.get('page', '1'))
    except ValueError:
        pageno = 1
    try :
        page = paginator.page(pageno)
    except (EmptyPage, InvalidPage):
        page = paginator.page(paginator.num_pages) #last page
    return page

class IssueVotesHandler(AnonymousBaseHandler):
    """
    Return votes for issue
    """
    allowed_methods = ('GET',)
    fields = ('vote', 'vote_count',)
    model = Vote

    def read(self, request, id, *args, **kwargs):
        """
        Return the different vote counts for an issue

        example output:

        { "-1": 1 , "1":10, "10": 1  ... }

        This Means for -1 , (against) there is 1 vote
        , 10 people voted for (1) and 1 person voted blank (10) *unconvincing*
        Here are all the different votes directions:

        -1.  "Against",

        1.   "For",

        *blank_votes* note always higer than 10.

        content related problems with issues

        10.  'Unconvincing',
        11.  'Not political',
        12.  'Can\'t completely agree',

        form related problems with issues

        13.  "Needs more work",
        14.  "Badly worded",
        15.  "Duplicate",
        16.  'Unrelated source',

        personal considerations

        17.  'I need to know more',
        18.  'Ask me later',
        19.  'Too personal',

        """
        try :
            issue = Issue.objects.get( id=id )
        except Issue.DoesNotExist :
            return rc.NOT_HERE

        votes = Vote.objects.get_for_object( issue )
        return votes

    @classmethod
    def vote(cls, vote):
        return vote[0]

    @classmethod
    def vote_count(cls, vote):
        return vote[1]

    @staticmethod
    def resource_uri():
        return ('api_issue_votes' , ['id'])

class IssueList(BaseHandler):
    """
    BaseClass for Listings of issues

    #. 'new'
    #. 'popular'
    #. 'controversial'
    #. 'recommended' #TODO
    #. 'with_tag'    #TODO

    """
    allowed_methods = ('GET',)

    def read(self, request , **kwargs):
        """ get listing of issue uri's """

        sortorder = kwargs.get("sortorder", "")
        if sortorder == 'popular':
            qs = Vote.objects.get_popular(Issue)
        elif sortorder == 'controversial':
            qs = Vote.objects.get_controversial(Issue)
        elif sortorder == 'new':
            qs = Issue.objects.all().order_by('-time_stamp')
            qs = qs.values_list('id', 'time_stamp')

        else:
            return rc.BAD_REQUEST
        page = paginate(request, qs )
        result = []
        for id_value in page.object_list:
            uri = self.issue_url( id_value )
            result.append( (uri, id_value[1]) )
        return result

    # not no classmethodis because query returns list of tuples.
    # and the responder will not look at those.
    def issue_url(cls, id_value ):
        return reverse('api_issue', args=[id_value[0]])

class VoteHandler(BaseHandler):
    """
    Read and Post votes for user.
    """
    allowed_methods = ('GET', 'POST', )
    fields = ('vote', 'time_stamp', 'issue_uri', 'keep_private', )
    model = Vote

    def read(self, request, id=None, **kwargs):
        """
        Returns the votes for an user.
        """
        # TODO check if args has issue id.
        ctype = ContentType.objects.get_for_model(Issue)
        queryset = self.model.objects.filter( user = request.user,
                        content_type = ctype.pk ).order_by('time_stamp')
        if id:
            queryset = queryset.filter( object_id=id )
        queryset.order_by( "username" )
        page = paginate(request, queryset)
        return page.object_list

    @classmethod
    def issue_uri(cls, vote):
        return "%s" % reverse('api_issue' , args=[vote.object_id])

    @validate( VoteForm )
    def create(self, request ):
        """
        Vote on an Issue
        """
        ctype = ContentType.objects.get_for_model(Issue)
        attrs = self.flatten_dict(request.POST)
        object_id = attrs.pop('issue')
        attrs.update({ 'content_type' : ctype,
                       'object_id' : object_id,
                })

        if self.exists(**attrs):
            return rc.DUPLICATE_ENTRY
        if not attrs.has_key('keep_private'):
            attrs['keep_private'] = False

        #print type(request.throttle_extra)

        gamelogic.actions.vote(
                request.user,
                Issue.objects.get(id=attrs['object_id']),
                int(attrs['vote']),
                attrs['keep_private'],
                api_interface=request.throttle_extra,
        )
        return rc.CREATED

class AnonymousUserHandler(AnonymousBaseHandler):
    allowed_methods = ('GET',)
    fields = ('username', 'score', 'ranking' , )
    model = User

    def read(self, request, id=None, *args, **kwargs):
        if id:
            try:
                return User.objects.filter( id=id )
            except User.DoesNotExist:
                return rc.NOT_FOUND
        else :
            queryset = User.objects.filter().order_by( 'username' )
            page = paginate(request, queryset)
            return page.object_list

    @classmethod
    def score(cls, user):
        return  get_profile_field(user, 'score', 0 )

    @classmethod
    def ranking(cls , user):
        score = cls.score(user)
        return UserProfile.objects.filter( score__gte = score ).count()

    #TODO GRAVATAR url !!
    @staticmethod
    def resource_uri():
        return ('api_user' , ['id'])

class UserHandler(BaseHandler):
    """
    Get User Details for authenticated user
    """
    allowed_methods = ('GET',)
    fields = ('username', 'firstname' , 'lastname' , 'email' , 'score', 'ranking' , 'role' )
    anonymous = AnonymousUserHandler
    model = User

    def read(self, request, *args, **kwargs):
        return request.user

    @classmethod
    def score(cls, user):
        return get_profile_field(user, 'score' , 0)

    @classmethod
    def role( cls , user):
        return get_profile_field(user, 'role', 'citizen' )

    @classmethod
    def ranking( cls , user):
        score = cls.score(user)
        return UserProfile.objects.filter( score__gte = score ).count()

    @staticmethod
    def resource_uri():
        return ('api_user' , ['id'])

class AnonymousIssueHandler(AnonymousBaseHandler):

    fields = ('title', 'body', ('user', ('username', ) ), 'time_stamp', 'source_type', 'url')
    model = Issue

    def read(self, request, id=None , *args, **kwargs):
        if id:
            try:
                return self.model.objects.filter( issue_id=id )
            except ObjectDoesNotExist:
                return rc.NOT_FOUND
        else :
            queryset = self.model.objects.order_by( '-time_stamp' )
            page = paginate(request, queryset)
            return page.object_list

        queryset = self.model.objects.filter()
        page = paginate(request, queryset)
        return page.object_list

    @staticmethod
    def resource_uri():
        return ('api_issue' , ['id'])

class IssueHandler(BaseHandler):
    allowed_methods = ('GET', 'POST',)
    anonymous = AnonymousIssueHandler
    fields = ('title', 'body', ('user', ('username')), 'time_stamp', 'souce_type', 'url')
    model = Issue

    @validate(IssueForm)
    def create(self, request):
        attrs = self.flatten_dict(request.POST)

        if self.exists(**attrs):
            return rc.DUPLICATE_ENTRY
        else :
            issue = gamelogic.actions.propose(
                request.user,
                attrs['title'],
                attrs['body'],
                attrs['direction'],
                attrs['url'],
                attrs['source_type'],
                attrs['is_draft'],
            )
            if issue : return rc.CREATED
            else : rc.BAD_REQUEST

    @staticmethod
    def resource_uri():
        return ('api_issue' , ['id'])

class AnonymousMultiplyHandler( AnonymousBaseHandler ):
    allowed_methods = ('GET',)
    model = MultiplyIssue

    fields = ( ('user', ('resource_uri') ), 'time_stamp' , ( 'issue' , ( 'resource_uri' ) ) )

    def read(self, request, id=None , *args, **kwargs):
        """ Read the active multiplies for specific issue
            or all multiplies in decending paginated order
        """
        if id:
            return self.model.objects.filter( issue=id )
        else :
            queryset = self.model.objects.all().order_by( '-time_stamp' )
            page = paginate(request, queryset)
            return page.object_list

        queryset = self.model.objects.filter()
        page = paginate(request, queryset)
        return page.object_list

    @staticmethod
    def resource_uri():
        return ('api_multiply' , ['id'])

class MultiplyHandler( BaseHandler ):
    """
    If a user has enough game points the user can multiply the value
    of an issue.
    """
    allowed_methods = ('POST' , 'GET' )
    anonymous = AnonymousMultiplyHandler
    model = MultiplyIssue
    fields = ( ('user', ('resource_uri') ), 'time_stamp' , ( 'issue' , ( 'resource_uri' ) ) )

    #def read , default behaviour is what we want.

    @validate(MultiplyForm)
    def create( self , request ):
        """ expects an issue id and optional a downgrade boolean
            in the POST data
        """
        attrs = self.flatten_dict(request.POST)

        issue = gamelogic.actions.multiply(
            request.user,
            Issue.objects.get( id = attrs['issue'] ) ,
            attrs.get('downgrade', False) ,
        )
        if issue : return rc.CREATED
        else : return rc.FORBIDDEN

    @staticmethod
    def resource_uri():
        return ('api_multiply' , ['id'])

# helper method.
def get_profile(user):
    try :
        return user.get_profile()
    except UserProfile.DoesNotExist:
        pass

def get_profile_field(user, attribute, default):
    profile = get_profile(user)
    if not profile: return default
    if hasattr(profile, attribute):
            return getattr(profile, attribute)
    return default
<commit_msg> fixed style issues with pep8.py<commit_after>"""
The democracy API.

we have some public resources which you can get annonymousy
but for the private data we require oauth authentication

"""
import datetime

from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.core.paginator import Paginator
from django.core.paginator import InvalidPage
from django.core.paginator import EmptyPage
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.contenttypes.models import ContentType

from piston.handler import BaseHandler, AnonymousBaseHandler
from piston.utils import rc
from piston.utils import validate

from democracy.api.forms import IssueForm
from democracy.api.forms import VoteForm
from democracy.api.forms import MultiplyForm
from democracy.issue.models import Issue
from democracy.voting.models import Vote
from democracy.profiles.models import UserProfile
from democracy.gamelogic.models import MultiplyIssue

import gamelogic.actions

from piston.models import Token

def paginate(request, qs):
    paginator = Paginator(qs, 3)  # TODO add to settings.py
    try:
        pageno = int(request.GET.get('page', '1'))
    except ValueError:
        pageno = 1
    try:
        page = paginator.page(pageno)
    except (EmptyPage, InvalidPage):
        page = paginator.page(paginator.num_pages) #last page
    return page

class IssueVotesHandler(AnonymousBaseHandler):
    """
    Return votes for issue
    """
    allowed_methods = ('GET', )
    fields = ('vote', 'vote_count', )
    model = Vote

    def read(self, request, id, *args, **kwargs):
        """
        Return the different vote counts for an issue

        example output:

        {"-1": 1, "1":10, "10": 1  ... }

        This Means for -1, (against) there is 1 vote
        , 10 people voted for (1) and 1 person voted blank (10) *unconvincing*
        Here are all the different votes directions:

        -1.  "Against",

        1.   "For",

        *blank_votes* note always higer than 10.

        content related problems with issues

        10.  'Unconvincing',
        11.  'Not political',
        12.  'Can\'t completely agree',

        form related problems with issues

        13.  "Needs more work",
        14.  "Badly worded",
        15.  "Duplicate",
        16.  'Unrelated source',

        personal considerations

        17.  'I need to know more',
        18.  'Ask me later',
        19.  'Too personal',

        """
        try:
            issue = Issue.objects.get(id=id)
        except Issue.DoesNotExist:
            return rc.NOT_HERE

        votes = Vote.objects.get_for_object(issue)
        return votes

    @classmethod
    def vote(cls, vote):
        return vote[0]

    @classmethod
    def vote_count(cls, vote):
        return vote[1]

    @staticmethod
    def resource_uri():
        return ('api_issue_votes', ['id'])

class IssueList(BaseHandler):
    """
    BaseClass for Listings of issues

    #. 'new'
    #. 'popular'
    #. 'controversial'
    #. 'recommended' #TODO
    #. 'with_tag'    #TODO

    """
    allowed_methods = ('GET', )

    def read(self, request, **kwargs):
        """ get listing of issue uri's """

        sortorder = kwargs.get("sortorder", "")
        if sortorder == 'popular':
            qs = Vote.objects.get_popular(Issue)
        elif sortorder == 'controversial':
            qs = Vote.objects.get_controversial(Issue)
        elif sortorder == 'new':
            qs = Issue.objects.all().order_by('-time_stamp')
            qs = qs.values_list('id', 'time_stamp')

        else:
            return rc.BAD_REQUEST
        page = paginate(request, qs)
        result = []
        for id_value in page.object_list:
            uri = self.issue_url(id_value)
            result.append((uri, id_value[1]))
        return result

    # not no classmethodis because query returns list of tuples.
    # and the responder will not look at those.
    def issue_url(cls, id_value):
        return reverse('api_issue', args=[id_value[0]])

class VoteHandler(BaseHandler):
    """
    Read and Post votes for user.
    """
    allowed_methods = ('GET', 'POST', )
    fields = ('vote', 'time_stamp', 'issue_uri', 'keep_private', )
    model = Vote

    def read(self, request, id=None, **kwargs):
        """
        Returns the votes for an user.
        """
        # TODO check if args has issue id.
        ctype = ContentType.objects.get_for_model(Issue)
        queryset = self.model.objects.filter(user = request.user,
                        content_type = ctype.pk).order_by('time_stamp')
        if id:
            queryset = queryset.filter(object_id=id)
        queryset.order_by("username")
        page = paginate(request, queryset)
        return page.object_list

    @classmethod
    def issue_uri(cls, vote):
        return "%s" % reverse('api_issue', args=[vote.object_id])

    @validate(VoteForm)
    def create(self, request):
        """
        Vote on an Issue
        """
        ctype = ContentType.objects.get_for_model(Issue)
        attrs = self.flatten_dict(request.POST)
        object_id = attrs.pop('issue')
        attrs.update({'content_type': ctype,
                       'object_id': object_id,
                })

        if self.exists(**attrs):
            return rc.DUPLICATE_ENTRY
        if not 'keep_private' in attrs:
            attrs['keep_private'] = False

        #print type(request.throttle_extra)

        gamelogic.actions.vote(
                request.user,
                Issue.objects.get(id=attrs['object_id']),
                int(attrs['vote']),
                attrs['keep_private'],
                api_interface=request.throttle_extra,
        )
        return rc.CREATED

class AnonymousUserHandler(AnonymousBaseHandler):
    allowed_methods = ('GET', )
    fields = ('username', 'score', 'ranking', )
    model = User

    def read(self, request, id=None, *args, **kwargs):
        if id:
            try:
                return User.objects.filter(id=id)
            except User.DoesNotExist:
                return rc.NOT_FOUND
        else:
            queryset = User.objects.filter().order_by('username')
            page = paginate(request, queryset)
            return page.object_list

    @classmethod
    def score(cls, user):
        return  get_profile_field(user, 'score', 0)

    @classmethod
    def ranking(cls, user):
        score = cls.score(user)
        return UserProfile.objects.filter(score__gte = score).count()

    #TODO GRAVATAR url !!
    @staticmethod
    def resource_uri():
        return ('api_user', ['id'])

class UserHandler(BaseHandler):
    """
    Get User Details for authenticated user
    """
    allowed_methods = ('GET', )
    fields = ('username', 'firstname', 'lastname', 'email', 'score',
            'ranking', 'role')
    anonymous = AnonymousUserHandler
    model = User

    def read(self, request, *args, **kwargs):
        return request.user

    @classmethod
    def score(cls, user):
        return get_profile_field(user, 'score', 0)

    @classmethod
    def role(cls, user):
        return get_profile_field(user, 'role', 'citizen')

    @classmethod
    def ranking(cls, user):
        score = cls.score(user)
        return UserProfile.objects.filter(score__gte = score).count()

    @staticmethod
    def resource_uri():
        return ('api_user', ['id'])

class AnonymousIssueHandler(AnonymousBaseHandler):

    fields = ('title', 'body', ('user', ('username', )), 'time_stamp',
            'source_type', 'url')
    model = Issue

    def read(self, request, id=None, *args, **kwargs):
        if id:
            try:
                return self.model.objects.filter(issue_id=id)
            except ObjectDoesNotExist:
                return rc.NOT_FOUND
        else:
            queryset = self.model.objects.order_by('-time_stamp')
            page = paginate(request, queryset)
            return page.object_list

        queryset = self.model.objects.filter()
        page = paginate(request, queryset)
        return page.object_list

    @staticmethod
    def resource_uri():
        return ('api_issue', ['id'])

class IssueHandler(BaseHandler):
    allowed_methods = ('GET', 'POST')
    anonymous = AnonymousIssueHandler
    fields = ('title', 'body', ('user', ('username')), 'time_stamp',
            'souce_type', 'url')
    model = Issue

    @validate(IssueForm)
    def create(self, request):
        attrs = self.flatten_dict(request.POST)

        if self.exists(**attrs):
            return rc.DUPLICATE_ENTRY
        else:
            issue = gamelogic.actions.propose(
                request.user,
                attrs['title'],
                attrs['body'],
                attrs['direction'],
                attrs['url'],
                attrs['source_type'],
                attrs['is_draft'],
           )
            if issue:
                return rc.CREATED
            else:
                return rc.BAD_REQUEST

    @staticmethod
    def resource_uri():
        return ('api_issue', ['id'])

class AnonymousMultiplyHandler(AnonymousBaseHandler):
    allowed_methods = ('GET', )
    model = MultiplyIssue

    fields = (('user', ('resource_uri')), 'time_stamp',
            ('issue', ('resource_uri')))

    def read(self, request, id=None, *args, **kwargs):
        """ Read the active multiplies for specific issue
            or all multiplies in decending paginated order
        """
        if id:
            return self.model.objects.filter(issue=id)
        else:
            queryset = self.model.objects.all().order_by('-time_stamp')
            page = paginate(request, queryset)
            return page.object_list

        queryset = self.model.objects.filter()
        page = paginate(request, queryset)
        return page.object_list

    @staticmethod
    def resource_uri():
        return ('api_multiply', ['id'])

class MultiplyHandler(BaseHandler):
    """
    If a user has enough game points the user can multiply the value
    of an issue.
    """
    allowed_methods = ('POST', 'GET')
    anonymous = AnonymousMultiplyHandler
    model = MultiplyIssue
    fields = (('user', ('resource_uri')), 'time_stamp', ('issue',
        ('resource_uri')))

    #No def read, default behaviour is what we want.
    @validate(MultiplyForm)
    def create(self, request):
        """ expects an issue id and optional a downgrade boolean
            in the POST data
        """
        attrs = self.flatten_dict(request.POST)

        issue = gamelogic.actions.multiply(
            request.user,
            Issue.objects.get(id = attrs['issue']),
            attrs.get('downgrade', False),
        )
        if issue:
            return rc.CREATED
        else:
            return rc.FORBIDDEN

    @staticmethod
    def resource_uri():
        return ('api_multiply', ['id'])

def get_profile(user):
    try:
        return user.get_profile()
    except UserProfile.DoesNotExist:
        pass

def get_profile_field(user, attribute, default):
    profile = get_profile(user)
    if not profile:
        return default
    if hasattr(profile, attribute):
            return getattr(profile, attribute)
    return default
<|endoftext|>


<commit_before>"""
This class will process deleted group members,
mark the user accounts in the database terminated.
Update netid if it has changed
"""

import logging
import traceback
from sis_provisioner.account_managers.gws_bridge import GwsBridgeLoader
from sis_provisioner.dao.gws import get_deleted_members
from sis_provisioner.dao.pws import get_person
from sis_provisioner.dao.uw_account import save_uw_account
from sis_provisioner.util.settings import get_group_member_del_window

logger = logging.getLogger(__name__)

class TerminateUser(GwsBridgeLoader):

    def __init__(self, worker, clogger=logger):
        super(TerminateUser, self).__init__(worker, clogger)
        self.data_source = "Group deleted member"

    def fetch_users(self):
        return list(get_deleted_members(get_group_member_del_window()))

    def process_users(self):
        for netid in self.get_users_to_process():
            person = get_person(netid)
            if self.is_invalid_person(netid, person):
                continue

            uw_acc = save_uw_account(person, create=False)
            if not uw_acc or uw_acc.disabled or uw_acc.has_terminate_date():
                continue

            # existing active user
            self.total_checked_users += 1
            uwnetid = uw_acc.netid
            if not self.in_uw_groups(uwnetid):
                logger.info(
                    "{} has left UW, schedule terminate".format(uw_acc))
                uw_acc.set_terminate_date(graceful=True)

            if uw_acc.netid_changed():
                try:
                    self.apply_change_to_bridge(uw_acc, person)
                except Exception as ex:
                    self.handle_exception(
                        "Failed to change netid {0} ".format(uwnetid),
                        ex, traceback)
<commit_msg>revise comment<commit_after>"""
This class will process deleted group members,
mark the user accounts in the database terminated.
Update netid if it has changed
"""

import logging
import traceback
from sis_provisioner.account_managers.gws_bridge import GwsBridgeLoader
from sis_provisioner.dao.gws import get_deleted_members
from sis_provisioner.dao.pws import get_person
from sis_provisioner.dao.uw_account import save_uw_account
from sis_provisioner.util.settings import get_group_member_del_window

logger = logging.getLogger(__name__)

class TerminateUser(GwsBridgeLoader):

    def __init__(self, worker, clogger=logger):
        super(TerminateUser, self).__init__(worker, clogger)
        self.data_source = "Group deleted member"

    def fetch_users(self):
        return list(get_deleted_members(get_group_member_del_window()))

    def process_users(self):
        for netid in self.get_users_to_process():
            person = get_person(netid)
            if self.is_invalid_person(netid, person):
                continue

            uw_acc = save_uw_account(person, create=False)
            if not uw_acc or uw_acc.disabled or uw_acc.has_terminate_date():
                continue

            # existing active user to mark terminate
            self.total_checked_users += 1
            uwnetid = uw_acc.netid
            if not self.in_uw_groups(uwnetid):
                logger.info(
                    "{} has left UW, schedule terminate".format(uw_acc))
                uw_acc.set_terminate_date(graceful=True)

            if uw_acc.netid_changed():
                try:
                    self.apply_change_to_bridge(uw_acc, person)
                except Exception as ex:
                    self.handle_exception(
                        "Failed to change netid {0} ".format(uwnetid),
                        ex, traceback)
<|endoftext|>


<commit_before>''' Script to check the correctness of the analysis. The analysis is done on raw data and all results are compared to a recorded analysis.
'''
import os

import unittest

import numpy as np

from testbeam_analysis import track_analysis
from testbeam_analysis.tools import test_tools
from pykalman.standard import KalmanFilter

# Get package path
testing_path = os.path.dirname(__file__)  # Get the absoulte path of the online_monitor installation

# Set the converter script path
tests_data_folder = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(testing_path)) + r'/testing/fixtures/track_analysis/'))

def _kalman_fit_3d(hits, dut_fit_selection, transition_matrix, transition_covariance, transition_offset, observation_matrix, observation_covariance, observation_offset, initial_state_mean, initial_state_covariance):
    kf = KalmanFilter(transition_matrix, observation_matrix,
                      transition_covariance, observation_covariance,
                      transition_offset, observation_offset,
                      initial_state_mean, initial_state_covariance)
    measurements = np.ma.array(np.c_[hits[:, 0:2], np.zeros((hits.shape[0])), np.zeros((hits.shape[0]))])  # x and y pos, slopes in y and y are zero

    # mask duts which should not used in fit
    for dut_index in range(0, hits.shape[0]):
        if dut_index not in dut_fit_selection:
            measurements[dut_index] = np.ma.masked

    smoothed_state_estimates, cov = kf.smooth(measurements)

    x_err = np.sqrt(np.diagonal(cov, axis1=2, axis2=1))[:, 0]
    y_err = np.sqrt(np.diagonal(cov, axis1=2, axis2=1))[:, 1]

    chi2 = np.sum(np.square(measurements[:, 0:2] - smoothed_state_estimates[:, 0:2]), dtype=np.uint32)

    return smoothed_state_estimates, chi2, x_err, y_err

def _fit_tracks_kalman_loop(track_hits, dut_fit_selection, pixel_size, z_positions, beam_energy, total_thickness, radiation_length):
    n_duts = track_hits.shape[1]
    dut_selection = np.array(range(0, n_duts))
    z_positions = np.array(z_positions)

    # create a list of duts which should be included in the fit
    dut_list = np.full(shape=(n_duts), fill_value=np.nan)
    for index in range(n_duts):
        dut_n = index
        if np.bitwise_and(1 << index, dut_fit_selection) == 2 ** index:
            dut_list[dut_n] = dut_n
    dut_fit_selection = dut_list[~np.isnan(dut_list)].astype(int)

    # set multiple scattering environment
    total_thickness = np.array(total_thickness)
    radiation_length = np.array(radiation_length)  # total radiation length of 2 kapton foils (before and after sensor) and m26 sensor in um

    pixel_resolution = pixel_size / np.sqrt(12.)  # Resolution of each telescope plane

    # Calculate multiple scattering
    mass = 0.511  # mass in MeV (electrons)
    momentum = np.sqrt(beam_energy * beam_energy - mass * mass)
    beta = momentum / beam_energy  # almost 1
    # rms angle of multiple scattering
    theta = np.array(((13.6 / momentum / beta) * np.sqrt(total_thickness / radiation_length) * (1. + 0.038 * np.log(total_thickness / radiation_length))))

    track_estimates_chunk = np.zeros((track_hits.shape[0], track_hits.shape[1], 4))
    chi2_tot = np.empty((track_hits.shape[0],), dtype=np.float)
    x_errs = np.zeros((track_hits.shape[0], track_hits.shape[1]))
    y_errs = np.zeros((track_hits.shape[0], track_hits.shape[1]))

    # express transition covariance matrix
    transition_covariance = np.zeros((n_duts - 1, 4, 4))

    # express transition matrix
    transition_matrix = np.zeros((n_duts - 1, 4, 4))

    # express state vector
    state_vector = np.zeros((n_duts - 1, 4,))

    # express transition and observation offset matrices
    transition_offset = np.zeros((n_duts - 1, 4))
    observation_offset = np.zeros((n_duts, 4))

    # express observatipon matrix
    observation_matrix = np.zeros((n_duts, 4, 4))

    # express observation covariance matrices
    observation_covariance = np.zeros((n_duts, 4, 4))
    # observation matrix: only observe x and y position
    observation_matrix[:, 0, 0] = 1.
    observation_matrix[:, 1, 1] = 1.
    # express initial state covariance matrices: x and y pos have initial error of pixel resolution and x and y slopes have large error
    initial_state_covariance = np.zeros((4, 4))
    initial_state_covariance[0, 0] = pixel_resolution[0, 0]**2
    initial_state_covariance[1, 1] = pixel_resolution[0, 1]**2
    initial_state_covariance[2, 2] = 0.01
    initial_state_covariance[3, 3] = 0.01

    for index, actual_hits in enumerate(track_hits):  # Loop over selected track candidate hits and fit
        x_positions = np.array(actual_hits[:, 0])
        y_positions = np.array(actual_hits[:, 1])

        # initial state: first two components x and y hit position in first dut, slopes all zero
        initial_state_mean = np.array([actual_hits[0, 0], actual_hits[0, 1], 0., 0.])

        # express observation covariance
        observation_covariance[dut_selection, 0, 0] = np.square(pixel_resolution[dut_selection][:, 0])
        observation_covariance[dut_selection, 1, 1] = np.square(pixel_resolution[dut_selection][:, 1])

        # express transition matrices
        transition_matrix[dut_selection[:-1], :, 0] = np.array([1., 0., 0., 0.])
        transition_matrix[dut_selection[:-1], :, 1] = np.array([0., 1., 0., 0.])
        # need to check wether the slope is negative or positive
        transition_matrix[dut_selection[:-1], :, 2] = np.array([-(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]), np.zeros((len(dut_selection[:-1]),)), np.ones((len(dut_selection[:-1]),)), np.zeros((len(dut_selection[:-1]),))]).T
        transition_matrix[dut_selection[:-1], :, 3] = np.array([np.zeros((len(dut_selection[:-1]),)), - (z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]), np.zeros((len(dut_selection[:-1]),)), np.ones((len(dut_selection[:-1]),))]).T

        state_vector[dut_fit_selection[:-1]] = np.array([x_positions[dut_fit_selection[:-1]],
                                                         y_positions[dut_fit_selection[:-1]],
                                                         np.zeros(x_positions[dut_fit_selection[:-1]].shape),
                                                         np.zeros(y_positions[dut_fit_selection[:-1]].shape)]).T

        cov_33 = theta[dut_selection[:-1]]**2 * (1 + state_vector[dut_selection[:-1], 2]**2) * (1 + state_vector[dut_selection[:-1], 2]**2 + state_vector[dut_selection[:-1], 3]**2).T
        cov_44 = theta[dut_selection[:-1]]**2 * (1 + state_vector[dut_selection[:-1], 3]**2) * (1 + state_vector[dut_selection[:-1], 2]**2 + state_vector[dut_selection[:-1], 3]**2).T
        cov_34 = theta[dut_selection[:-1]]**2 * state_vector[dut_selection[:-1], 2] * state_vector[dut_selection[:-1], 3] * (1 + state_vector[dut_selection[:-1], 2]**2 + state_vector[dut_selection[:-1], 3]**2).T

        # express transition covariance matrices, according to http://web-docs.gsi.de/~ikisel/reco/Methods/CovarianceMatrices-NIMA329-1993.pdf
        transition_covariance[dut_selection[:-1], :, 0] = np.array([(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]])**2 * cov_33,
                                                                    (z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]])**2 * cov_34,
                                                                    -(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]) * cov_33,
                                                                    -(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]) * cov_34]).T

        transition_covariance[dut_selection[:-1], :, 1] = np.array([(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]])**2 * cov_34,
                                                                    (z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]])**2 * cov_44,
                                                                    -(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]) * cov_34,
                                                                    -(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]) * cov_44]).T
        transition_covariance[dut_selection[:-1], :, 2] = np.array([-(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]) * cov_33,
                                                                    -(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]) * cov_34,
                                                                    cov_33,
                                                                    cov_34]).T
        transition_covariance[dut_selection[:-1], :, 3] = np.array([-(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]) * cov_34,
                                                                    -(z_positions[dut_selection[:-1] + 1] - z_positions[dut_selection[:-1]]) * cov_44,
                                                                    cov_34,
                                                                    cov_44]).T
        # run kalman filter
        track_estimate_chunk, chi2, x_err, y_err = _kalman_fit_3d(actual_hits, dut_fit_selection,
                                                                  transition_matrix, transition_covariance,
                                                                  transition_offset, observation_matrix,
                                                                  observation_covariance, observation_offset,
                                                                  initial_state_mean, initial_state_covariance)

        chi2_tot[index] = chi2
        track_estimates_chunk[index] = track_estimate_chunk
        x_errs[index] = x_err
        y_errs[index] = y_err

    return track_estimates_chunk, chi2_tot, x_errs, y_errs

class TestTrackAnalysis(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        if os.getenv('TRAVIS', False):
            from xvfbwrapper import Xvfb  # virtual X server for plots under headless LINUX travis testing is needed
            cls.vdisplay = Xvfb()
            cls.vdisplay.start()
        # cls.output_folder = tests_data_folder
        # cls.pixel_size = (250, 50)  # in um

    @classmethod
    def tearDownClass(cls):  # Remove created files
        pass
        # os.remove(os.path.join(cls.output_folder, 'Tracks_merged.pdf'))

    def test_kalman(self):
        kwargs = {'track_hits': np.array([[[-1229.22372954, 2828.19616302, 0.],
                                           [np.nan, np.nan, np.nan],  # [-1254.51224282, 2827.4291421, 29900.],
                                           [-1285.6117892, 2822.34536687, 60300.],
                                           [-1311.31083616, 2823.56121414, 82100.],
                                           [-1335.8529645, 2828.43359043, 118700.],
                                           [-1357.81872222, 2840.86947964, 160700.],
                                           [-1396.35698339, 2843.76799577, 197800.]]]),
                  'dut_fit_selection': 61,
                  'z_positions': [[0., 29900, 60300, 82100, 118700, 160700, 197800]],
                  'pixel_size': ((18.5, 18.5), (18.5, 18.5), (18.5, 18.5),
                                 (18.5, 18.5), (18.5, 18.5), (250, 50), (250, 50)),
                  'beam_energy': 2500.,
                  'total_thickness': [[100., 100., 100., 100., 100., 100., 250.]],
                  'radiation_length': [[125390., 125390., 125390., 125390., 125390., 125390., 93700.]]}
        for i in range(4):  # test each return (state estimates, chi, x error, y errors) seperatly
            test = test_tools._call_function_with_args(function=track_analysis._fit_tracks_kalman_loop,
                                                       **kwargs)[0][i]
            orig = test_tools._call_function_with_args(function=_fit_tracks_kalman_loop,
                                                       **kwargs)[0][i]
            data_equal = test_tools.array_close(test, orig)
            self.assertTrue(data_equal)

if __name__ == '__main__':
    import logging
    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - [%(levelname)-8s] (%(threadName)-10s) %(message)s")
    suite = unittest.TestLoader().loadTestsFromTestCase(TestTrackAnalysis)
    unittest.TextTestRunner(verbosity=2).run(suite)
<commit_msg>TST: do not test against library, instead test against result array<commit_after>''' Script to check the correctness of the analysis. The analysis is done on raw data and all results are compared to a recorded analysis.
'''
import os

import unittest

import numpy as np

from testbeam_analysis import track_analysis
from testbeam_analysis.tools import test_tools

# Get package path
testing_path = os.path.dirname(__file__)  # Get the absoulte path of the online_monitor installation

# Set the converter script path
tests_data_folder = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(testing_path)) + r'/testing/fixtures/track_analysis/'))

class TestTrackAnalysis(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        if os.getenv('TRAVIS', False):
            from xvfbwrapper import Xvfb  # virtual X server for plots under headless LINUX travis testing is needed
            cls.vdisplay = Xvfb()
            cls.vdisplay.start()
        # cls.output_folder = tests_data_folder
        # cls.pixel_size = (250, 50)  # in um

    @classmethod
    def tearDownClass(cls):  # Remove created files
        pass
        # os.remove(os.path.join(cls.output_folder, 'Tracks_merged.pdf'))

    def test_kalman(self):
        kwargs = {'track_hits': np.array([[[-1229.22372954, 2828.19616302, 0.],
                                           [np.nan, np.nan, np.nan],  # [-1254.51224282, 2827.4291421, 29900.],
                                           [-1285.6117892, 2822.34536687, 60300.],
                                           [-1311.31083616, 2823.56121414, 82100.],
                                           [-1335.8529645, 2828.43359043, 118700.],
                                           [-1357.81872222, 2840.86947964, 160700.],
                                           [-1396.35698339, 2843.76799577, 197800.]]]),
                  'dut_fit_selection': 61,
                  'z_positions': [[0., 29900, 60300, 82100, 118700, 160700, 197800]],
                  'pixel_size': ((18.5, 18.5), (18.5, 18.5), (18.5, 18.5),
                                 (18.5, 18.5), (18.5, 18.5), (250, 50), (250, 50)),
                  'beam_energy': 2500.,
                  'total_thickness': [[100., 100., 100., 100., 100., 100., 250.]],
                  'radiation_length': [[125390., 125390., 125390., 125390., 125390., 125390., 93700.]]}

        # expected result array: (state estimates, chi, x error, y errors)
        result = np.array([[[-1.22922375e+03, 2.82819616e+03, 9.53048967e-04, 5.96680029e-05],
                            [-1.25771995e+03, 2.82641209e+03, 9.53050162e-04, 5.96680777e-05],
                            [-1.28669270e+03, 2.82459818e+03, 9.38989287e-04, 2.97608805e-05],
                            [-1.30661393e+03, 2.82511648e+03, 9.13818199e-04, -2.37752913e-05],
                            [-1.33842616e+03, 2.82786418e+03, 8.69186632e-04, -7.50739214e-05],
                            [-1.37485963e+03, 2.83196293e+03, 8.67463483e-04, -9.75893170e-05],
                            [-1.40704252e+03, 2.83558350e+03, 8.67463483e-04, -9.75893170e-05]],
                          [405.],
                          [3.77629595, 3.00521807, 3.22523435, 3.20152958, 4.56468792, 9.49981871, 15.38928532],
                          [3.77629595, 3.00434937, 3.22194104, 3.1832628,  4.10202498, 7.98372842, 13.19036558]])
        for i in range(4):  # test each return (state estimates, chi, x error, y errors) seperatly
            test = test_tools._call_function_with_args(function=track_analysis._fit_tracks_kalman_loop,
                                                       **kwargs)[0][i]
            data_equal = test_tools.array_close(test, result[i])
            self.assertTrue(data_equal)

if __name__ == '__main__':
    import logging
    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - [%(levelname)-8s] (%(threadName)-10s) %(message)s")
    suite = unittest.TestLoader().loadTestsFromTestCase(TestTrackAnalysis)
    unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys

import gfxprim.core as core
import gfxprim.loaders as loaders
import gfxprim.filters as filters

def main():
    if len(sys.argv) != 2:
        print("usage: dither.py image")
        sys.exit(1)

    # Load Image
    img = loaders.Load(sys.argv[1])
    # Use Floyd-Steinberg dithering
    res = filters.FilterFloydSteinberg_RGB888_Alloc(img, core.C.PIXEL_G1, None)
    # Save result into grayscale png
    loaders.SavePNG(res, "out.png", None)

if __name__ == '__main__':
    main()
<commit_msg>demos: py_simple: Make use of loaders submodule.<commit_after>#!/usr/bin/env python
import sys

import gfxprim.core as core
import gfxprim.loaders as loaders
import gfxprim.filters as filters

def main():
    if len(sys.argv) != 2:
        print("usage: dither.py image")
        sys.exit(1)

    # Load Image
    img = loaders.Load(sys.argv[1])
    # Use Floyd-Steinberg dithering
    res = filters.FilterFloydSteinberg_RGB888_Alloc(img, core.C.PIXEL_G1, None)
    # Save result into grayscale png
    res.loaders.SavePNG("out.png")

if __name__ == '__main__':
    main()
<|endoftext|>