<commit_before>"""
 This module contains the spatial lookup types, and the get_geo_where_clause()
 routine for PostGIS.
"""
import re
from decimal import Decimal
from django.db import connection
from django.contrib.gis.measure import Distance
from django.contrib.gis.db.backend.postgis.management import postgis_version_tuple
from django.contrib.gis.db.backend.util import SpatialOperation, SpatialFunction
qn = connection.ops.quote_name

# Getting the PostGIS version information
POSTGIS_VERSION, MAJOR_VERSION, MINOR_VERSION1, MINOR_VERSION2 = postgis_version_tuple()

# The supported PostGIS versions.
#  TODO: Confirm tests with PostGIS versions 1.1.x -- should work.
#        Versions <= 1.0.x do not use GEOS C API, and will not be supported.
if MAJOR_VERSION != 1 or (MAJOR_VERSION == 1 and MINOR_VERSION1 < 1):
    raise Exception('PostGIS version %s not supported.' % POSTGIS_VERSION)

# Versions of PostGIS >= 1.2.2 changed their naming convention to be
#  'SQL-MM-centric' to conform with the ISO standard. Practically, this
#  means that 'ST_' prefixes geometry function names.
GEOM_FUNC_PREFIX = ''
if MAJOR_VERSION >= 1:
    if (MINOR_VERSION1 > 2 or
        (MINOR_VERSION1 == 2 and MINOR_VERSION2 >= 2)):
        GEOM_FUNC_PREFIX = 'ST_'

    def get_func(func): return '%s%s' % (GEOM_FUNC_PREFIX, func)

    # Custom selection not needed for PostGIS because GEOS geometries are
    # instantiated directly from the HEXEWKB returned by default.  If
    # WKT is needed for some reason in the future, this value may be changed,
    # e.g,, 'AsText(%s)'.
    GEOM_SELECT = None

    # Functions used by the GeoManager & GeoQuerySet
    AREA = get_func('Area')
    ASGEOJSON = get_func('AsGeoJson')
    ASKML = get_func('AsKML')
    ASGML = get_func('AsGML')
    ASSVG = get_func('AsSVG')
    CENTROID = get_func('Centroid')
    COLLECT = get_func('Collect')
    DIFFERENCE = get_func('Difference')
    DISTANCE = get_func('Distance')
    DISTANCE_SPHERE = get_func('distance_sphere')
    DISTANCE_SPHEROID = get_func('distance_spheroid')
    ENVELOPE = get_func('Envelope')
    EXTENT = get_func('extent')
    GEOM_FROM_TEXT = get_func('GeomFromText')
    GEOM_FROM_WKB = get_func('GeomFromWKB')
    INTERSECTION = get_func('Intersection')
    LENGTH = get_func('Length')
    LENGTH_SPHEROID = get_func('length_spheroid')
    MAKE_LINE = get_func('MakeLine')
    MEM_SIZE = get_func('mem_size')
    NUM_GEOM = get_func('NumGeometries')
    NUM_POINTS = get_func('npoints')
    PERIMETER = get_func('Perimeter')
    POINT_ON_SURFACE = get_func('PointOnSurface')
    SCALE = get_func('Scale')
    SNAP_TO_GRID = get_func('SnapToGrid')
    SYM_DIFFERENCE = get_func('SymDifference')
    TRANSFORM = get_func('Transform')
    TRANSLATE = get_func('Translate')

    # Special cases for union, KML, and GeoJSON methods.
    if MINOR_VERSION1 < 3:
        UNIONAGG = 'GeomUnion'
        UNION = 'Union'
    else:
        UNIONAGG = 'ST_Union'
        UNION = 'ST_Union'

    if MINOR_VERSION1 == 1:
        ASKML = False

    # Only 1.3.4+ have AsGeoJson.
    if (MINOR_VERSION1 < 3 or
        (MINOR_VERSION1 == 3 and MINOR_VERSION2 < 4)):
        ASGEOJSON = False
else:
    raise NotImplementedError('PostGIS versions < 1.0 are not supported.')

#### Classes used in constructing PostGIS spatial SQL ####
class PostGISOperator(SpatialOperation):
    "For PostGIS operators (e.g. `&&`, `~`)."
    def __init__(self, operator):
        super(PostGISOperator, self).__init__(operator=operator, beg_subst='%s %s %%s')

class PostGISFunction(SpatialFunction):
    "For PostGIS function calls (e.g., `ST_Contains(table, geom)`)."
    def __init__(self, function, **kwargs):
        super(PostGISFunction, self).__init__(get_func(function), **kwargs)

class PostGISFunctionParam(PostGISFunction):
    "For PostGIS functions that take another parameter (e.g. DWithin, Relate)."
    def __init__(self, func):
        super(PostGISFunctionParam, self).__init__(func, end_subst=', %%s)')

class PostGISDistance(PostGISFunction):
    "For PostGIS distance operations."
    dist_func = 'Distance'
    def __init__(self, operator):
        super(PostGISDistance, self).__init__(self.dist_func, end_subst=') %s %s',
                                              operator=operator, result='%%s')

class PostGISSpheroidDistance(PostGISFunction):
    "For PostGIS spherical distance operations (using the spheroid)."
    dist_func = 'distance_spheroid'
    def __init__(self, operator):
        # An extra parameter in `end_subst` is needed for the spheroid string.
        super(PostGISSpheroidDistance, self).__init__(self.dist_func,
                                                      beg_subst='%s(%s, %%s, %%s',
                                                      end_subst=') %s %s',
                                                      operator=operator, result='%%s')

class PostGISSphereDistance(PostGISFunction):
    "For PostGIS spherical distance operations."
    dist_func = 'distance_sphere'
    def __init__(self, operator):
        super(PostGISSphereDistance, self).__init__(self.dist_func, end_subst=') %s %s',
                                                    operator=operator, result='%%s')

class PostGISRelate(PostGISFunctionParam):
    "For PostGIS Relate(<geom>, <pattern>) calls."
    pattern_regex = re.compile(r'^[012TF\*]{9}$')
    def __init__(self, pattern):
        if not self.pattern_regex.match(pattern):
            raise ValueError('Invalid intersection matrix pattern "%s".' % pattern)
        super(PostGISRelate, self).__init__('Relate')

#### Lookup type mapping dictionaries of PostGIS operations. ####

# PostGIS-specific operators. The commented descriptions of these
# operators come from Section 6.2.2 of the official PostGIS documentation.
POSTGIS_OPERATORS = {
    # The "&<" operator returns true if A's bounding box overlaps or
    #  is to the left of B's bounding box.
    'overlaps_left' : PostGISOperator('&<'),
    # The "&>" operator returns true if A's bounding box overlaps or
    #  is to the right of B's bounding box.
    'overlaps_right' : PostGISOperator('&>'),
    # The "<<" operator returns true if A's bounding box is strictly
    #  to the left of B's bounding box.
    'left' : PostGISOperator('<<'),
    # The ">>" operator returns true if A's bounding box is strictly
    #  to the right of B's bounding box.
    'right' : PostGISOperator('>>'),
    # The "&<|" operator returns true if A's bounding box overlaps or
    #  is below B's bounding box.
    'overlaps_below' : PostGISOperator('&<|'),
    # The "|&>" operator returns true if A's bounding box overlaps or
    #  is above B's bounding box.
    'overlaps_above' : PostGISOperator('|&>'),
    # The "<<|" operator returns true if A's bounding box is strictly
    #  below B's bounding box.
    'strictly_below' : PostGISOperator('<<|'),
    # The "|>>" operator returns true if A's bounding box is strictly
    # above B's bounding box.
    'strictly_above' : PostGISOperator('|>>'),
    # The "~=" operator is the "same as" operator. It tests actual
    #  geometric equality of two features. So if A and B are the same feature,
    #  vertex-by-vertex, the operator returns true.
    'same_as' : PostGISOperator('~='),
    'exact' : PostGISOperator('~='),
    # The "@" operator returns true if A's bounding box is completely contained
    #  by B's bounding box.
    'contained' : PostGISOperator('@'),
    # The "~" operator returns true if A's bounding box completely contains
    #  by B's bounding box.
    'bbcontains' : PostGISOperator('~'),
    # The "&&" operator returns true if A's bounding box overlaps
    #  B's bounding box.
    'bboverlaps' : PostGISOperator('&&'),
    }

# For PostGIS >= 1.2.2 the following lookup types will do a bounding box query
# first before calling the more computationally expensive GEOS routines (called
# "inline index magic"):
# 'touches', 'crosses', 'contains', 'intersects', 'within', 'overlaps', and
# 'covers'.
POSTGIS_GEOMETRY_FUNCTIONS = {
    'equals' : PostGISFunction('Equals'),
    'disjoint' : PostGISFunction('Disjoint'),
    'touches' : PostGISFunction('Touches'),
    'crosses' : PostGISFunction('Crosses'),
    'within' : PostGISFunction('Within'),
    'overlaps' : PostGISFunction('Overlaps'),
    'contains' : PostGISFunction('Contains'),
    'intersects' : PostGISFunction('Intersects'),
    'relate' : (PostGISRelate, basestring),
    }

# Valid distance types and substitutions
dtypes = (Decimal, Distance, float, int, long)
def get_dist_ops(operator):
    "Returns operations for both regular and spherical distances."
    return (PostGISDistance(operator), PostGISSphereDistance(operator), PostGISSpheroidDistance(operator))
DISTANCE_FUNCTIONS = {
    'distance_gt' : (get_dist_ops('>'), dtypes),
    'distance_gte' : (get_dist_ops('>='), dtypes),
    'distance_lt' : (get_dist_ops('<'), dtypes),
    'distance_lte' : (get_dist_ops('<='), dtypes),
    }

if GEOM_FUNC_PREFIX == 'ST_':
    # The ST_DWithin, ST_CoveredBy, and ST_Covers routines become available in 1.2.2+
    POSTGIS_GEOMETRY_FUNCTIONS.update(
        {'coveredby' : PostGISFunction('CoveredBy'),
         'covers' : PostGISFunction('Covers'),
         })
    DISTANCE_FUNCTIONS['dwithin'] = (PostGISFunctionParam('DWithin'), dtypes)

# Distance functions are a part of PostGIS geometry functions.
POSTGIS_GEOMETRY_FUNCTIONS.update(DISTANCE_FUNCTIONS)

# Any other lookup types that do not require a mapping.
MISC_TERMS = ['isnull']

# These are the PostGIS-customized QUERY_TERMS -- a list of the lookup types
#  allowed for geographic queries.
POSTGIS_TERMS = POSTGIS_OPERATORS.keys() # Getting the operators first
POSTGIS_TERMS += POSTGIS_GEOMETRY_FUNCTIONS.keys() # Adding on the Geometry Functions
POSTGIS_TERMS += MISC_TERMS # Adding any other miscellaneous terms (e.g., 'isnull')
POSTGIS_TERMS = dict((term, None) for term in POSTGIS_TERMS) # Making a dictionary for fast lookups

# For checking tuple parameters -- not very pretty but gets job done.
def exactly_two(val): return val == 2
def two_to_three(val): return val >= 2 and val <=3
def num_params(lookup_type, val):
    if lookup_type in DISTANCE_FUNCTIONS and lookup_type != 'dwithin': return two_to_three(val)
    else: return exactly_two(val)

#### The `get_geo_where_clause` function for PostGIS. ####
def get_geo_where_clause(table_alias, name, lookup_type, geo_annot):
    "Returns the SQL WHERE clause for use in PostGIS SQL construction."
    # Getting the quoted field as `geo_col`.
    geo_col = '%s.%s' % (qn(table_alias), qn(name))
    if lookup_type in POSTGIS_OPERATORS:
        # See if a PostGIS operator matches the lookup type.
        return POSTGIS_OPERATORS[lookup_type].as_sql(geo_col)
    elif lookup_type in POSTGIS_GEOMETRY_FUNCTIONS:
        # See if a PostGIS geometry function matches the lookup type.
        tmp = POSTGIS_GEOMETRY_FUNCTIONS[lookup_type]

        # Lookup types that are tuples take tuple arguments, e.g., 'relate' and
        # distance lookups.
        if isinstance(tmp, tuple):
            # First element of tuple is the PostGISOperation instance, and the
            # second element is either the type or a tuple of acceptable types
            # that may passed in as further parameters for the lookup type.
            op, arg_type = tmp

            # Ensuring that a tuple _value_ was passed in from the user
            if not isinstance(geo_annot.value, (tuple, list)):
                raise TypeError('Tuple required for `%s` lookup type.' % lookup_type)

            # Number of valid tuple parameters depends on the lookup type.
            nparams = len(geo_annot.value)
            if not num_params(lookup_type, nparams):
                raise ValueError('Incorrect number of parameters given for `%s` lookup type.' % lookup_type)

            # Ensuring the argument type matches what we expect.
            if not isinstance(geo_annot.value[1], arg_type):
                raise TypeError('Argument type should be %s, got %s instead.' % (arg_type, type(geo_annot.value[1])))

            # For lookup type `relate`, the op instance is not yet created (has
            # to be instantiated here to check the pattern parameter).
            if lookup_type == 'relate':
                op = op(geo_annot.value[1])
            elif lookup_type in DISTANCE_FUNCTIONS and lookup_type != 'dwithin':
                if geo_annot.geodetic:
                    # Geodetic distances are only availble from Points to PointFields.
                    if geo_annot.geom_type != 'POINT':
                        raise TypeError('PostGIS spherical operations are only valid on PointFields.')
                    if geo_annot.value[0].geom_typeid != 0:
                        raise TypeError('PostGIS geometry distance parameter is required to be of type Point.')
                    # Setting up the geodetic operation appropriately.
                    if nparams == 3 and geo_annot.value[2] == 'spheroid': op = op[2]
                    else: op = op[1]
                else:
                    op = op[0]
        else:
            op = tmp
        # Calling the `as_sql` function on the operation instance.
        return op.as_sql(geo_col)
    elif lookup_type == 'isnull':
        # Handling 'isnull' lookup type
        return "%s IS %sNULL" % (geo_col, (not geo_annot.value and 'NOT ' or ''))

    raise TypeError("Got invalid lookup_type: %s" % repr(lookup_type))
<commit_msg>Added experimental support for a POSTGIS_VERSION setting for GeoDjango. If set, it will tell GeoDjango not to do the postgis_lib_version() query every time the server starts up (assuming a PostGIS backend). This is intentionally undocumented because Justin mentioned he might be refactoring the backends for Django 1.2 such that this setting would be unnecessary. Until then, people can use this hook.<commit_after>"""
 This module contains the spatial lookup types, and the get_geo_where_clause()
 routine for PostGIS.
"""

import re
from decimal import Decimal
from django.db import connection
from django.conf import settings
from django.contrib.gis.measure import Distance
from django.contrib.gis.db.backend.util import SpatialOperation, SpatialFunction

qn = connection.ops.quote_name

# Get the PostGIS version information.
# To avoid the need to do a database query to determine the PostGIS version
# each time the server starts up, one can optionally specify a
# POSTGIS_VERSION setting. This setting is intentionally undocumented and
# should be considered experimental, because an upcoming GIS backend
# refactoring might remove the need for it.
if hasattr(settings, 'POSTGIS_VERSION') and settings.POSTGIS_VERSION is not None:
    version_tuple = settings.POSTGIS_VERSION
else:
    # This import is intentionally within the 'else' so that it isn't executed
    # if the POSTGIS_VERSION setting is available.
    from django.contrib.gis.db.backend.postgis.management import postgis_version_tuple
    version_tuple = postgis_version_tuple()
POSTGIS_VERSION, MAJOR_VERSION, MINOR_VERSION1, MINOR_VERSION2 = version_tuple

# The supported PostGIS versions.
#  TODO: Confirm tests with PostGIS versions 1.1.x -- should work.
#        Versions <= 1.0.x do not use GEOS C API, and will not be supported.
if MAJOR_VERSION != 1 or (MAJOR_VERSION == 1 and MINOR_VERSION1 < 1):
    raise Exception('PostGIS version %s not supported.' % POSTGIS_VERSION)

# Versions of PostGIS >= 1.2.2 changed their naming convention to be
#  'SQL-MM-centric' to conform with the ISO standard. Practically, this
#  means that 'ST_' prefixes geometry function names.
GEOM_FUNC_PREFIX = ''
if MAJOR_VERSION >= 1:
    if (MINOR_VERSION1 > 2 or
        (MINOR_VERSION1 == 2 and MINOR_VERSION2 >= 2)):
        GEOM_FUNC_PREFIX = 'ST_'

    def get_func(func): return '%s%s' % (GEOM_FUNC_PREFIX, func)

    # Custom selection not needed for PostGIS because GEOS geometries are
    # instantiated directly from the HEXEWKB returned by default.  If
    # WKT is needed for some reason in the future, this value may be changed,
    # e.g,, 'AsText(%s)'.
    GEOM_SELECT = None

    # Functions used by the GeoManager & GeoQuerySet
    AREA = get_func('Area')
    ASGEOJSON = get_func('AsGeoJson')
    ASKML = get_func('AsKML')
    ASGML = get_func('AsGML')
    ASSVG = get_func('AsSVG')
    CENTROID = get_func('Centroid')
    COLLECT = get_func('Collect')
    DIFFERENCE = get_func('Difference')
    DISTANCE = get_func('Distance')
    DISTANCE_SPHERE = get_func('distance_sphere')
    DISTANCE_SPHEROID = get_func('distance_spheroid')
    ENVELOPE = get_func('Envelope')
    EXTENT = get_func('extent')
    GEOM_FROM_TEXT = get_func('GeomFromText')
    GEOM_FROM_WKB = get_func('GeomFromWKB')
    INTERSECTION = get_func('Intersection')
    LENGTH = get_func('Length')
    LENGTH_SPHEROID = get_func('length_spheroid')
    MAKE_LINE = get_func('MakeLine')
    MEM_SIZE = get_func('mem_size')
    NUM_GEOM = get_func('NumGeometries')
    NUM_POINTS = get_func('npoints')
    PERIMETER = get_func('Perimeter')
    POINT_ON_SURFACE = get_func('PointOnSurface')
    SCALE = get_func('Scale')
    SNAP_TO_GRID = get_func('SnapToGrid')
    SYM_DIFFERENCE = get_func('SymDifference')
    TRANSFORM = get_func('Transform')
    TRANSLATE = get_func('Translate')

    # Special cases for union, KML, and GeoJSON methods.
    if MINOR_VERSION1 < 3:
        UNIONAGG = 'GeomUnion'
        UNION = 'Union'
    else:
        UNIONAGG = 'ST_Union'
        UNION = 'ST_Union'

    if MINOR_VERSION1 == 1:
        ASKML = False

    # Only 1.3.4+ have AsGeoJson.
    if (MINOR_VERSION1 < 3 or
        (MINOR_VERSION1 == 3 and MINOR_VERSION2 < 4)):
        ASGEOJSON = False
else:
    raise NotImplementedError('PostGIS versions < 1.0 are not supported.')

#### Classes used in constructing PostGIS spatial SQL ####
class PostGISOperator(SpatialOperation):
    "For PostGIS operators (e.g. `&&`, `~`)."
    def __init__(self, operator):
        super(PostGISOperator, self).__init__(operator=operator, beg_subst='%s %s %%s')

class PostGISFunction(SpatialFunction):
    "For PostGIS function calls (e.g., `ST_Contains(table, geom)`)."
    def __init__(self, function, **kwargs):
        super(PostGISFunction, self).__init__(get_func(function), **kwargs)

class PostGISFunctionParam(PostGISFunction):
    "For PostGIS functions that take another parameter (e.g. DWithin, Relate)."
    def __init__(self, func):
        super(PostGISFunctionParam, self).__init__(func, end_subst=', %%s)')

class PostGISDistance(PostGISFunction):
    "For PostGIS distance operations."
    dist_func = 'Distance'
    def __init__(self, operator):
        super(PostGISDistance, self).__init__(self.dist_func, end_subst=') %s %s',
                                              operator=operator, result='%%s')

class PostGISSpheroidDistance(PostGISFunction):
    "For PostGIS spherical distance operations (using the spheroid)."
    dist_func = 'distance_spheroid'
    def __init__(self, operator):
        # An extra parameter in `end_subst` is needed for the spheroid string.
        super(PostGISSpheroidDistance, self).__init__(self.dist_func,
                                                      beg_subst='%s(%s, %%s, %%s',
                                                      end_subst=') %s %s',
                                                      operator=operator, result='%%s')

class PostGISSphereDistance(PostGISFunction):
    "For PostGIS spherical distance operations."
    dist_func = 'distance_sphere'
    def __init__(self, operator):
        super(PostGISSphereDistance, self).__init__(self.dist_func, end_subst=') %s %s',
                                                    operator=operator, result='%%s')

class PostGISRelate(PostGISFunctionParam):
    "For PostGIS Relate(<geom>, <pattern>) calls."
    pattern_regex = re.compile(r'^[012TF\*]{9}$')
    def __init__(self, pattern):
        if not self.pattern_regex.match(pattern):
            raise ValueError('Invalid intersection matrix pattern "%s".' % pattern)
        super(PostGISRelate, self).__init__('Relate')

#### Lookup type mapping dictionaries of PostGIS operations. ####

# PostGIS-specific operators. The commented descriptions of these
# operators come from Section 6.2.2 of the official PostGIS documentation.
POSTGIS_OPERATORS = {
    # The "&<" operator returns true if A's bounding box overlaps or
    #  is to the left of B's bounding box.
    'overlaps_left' : PostGISOperator('&<'),
    # The "&>" operator returns true if A's bounding box overlaps or
    #  is to the right of B's bounding box.
    'overlaps_right' : PostGISOperator('&>'),
    # The "<<" operator returns true if A's bounding box is strictly
    #  to the left of B's bounding box.
    'left' : PostGISOperator('<<'),
    # The ">>" operator returns true if A's bounding box is strictly
    #  to the right of B's bounding box.
    'right' : PostGISOperator('>>'),
    # The "&<|" operator returns true if A's bounding box overlaps or
    #  is below B's bounding box.
    'overlaps_below' : PostGISOperator('&<|'),
    # The "|&>" operator returns true if A's bounding box overlaps or
    #  is above B's bounding box.
    'overlaps_above' : PostGISOperator('|&>'),
    # The "<<|" operator returns true if A's bounding box is strictly
    #  below B's bounding box.
    'strictly_below' : PostGISOperator('<<|'),
    # The "|>>" operator returns true if A's bounding box is strictly
    # above B's bounding box.
    'strictly_above' : PostGISOperator('|>>'),
    # The "~=" operator is the "same as" operator. It tests actual
    #  geometric equality of two features. So if A and B are the same feature,
    #  vertex-by-vertex, the operator returns true.
    'same_as' : PostGISOperator('~='),
    'exact' : PostGISOperator('~='),
    # The "@" operator returns true if A's bounding box is completely contained
    #  by B's bounding box.
    'contained' : PostGISOperator('@'),
    # The "~" operator returns true if A's bounding box completely contains
    #  by B's bounding box.
    'bbcontains' : PostGISOperator('~'),
    # The "&&" operator returns true if A's bounding box overlaps
    #  B's bounding box.
    'bboverlaps' : PostGISOperator('&&'),
    }

# For PostGIS >= 1.2.2 the following lookup types will do a bounding box query
# first before calling the more computationally expensive GEOS routines (called
# "inline index magic"):
# 'touches', 'crosses', 'contains', 'intersects', 'within', 'overlaps', and
# 'covers'.
POSTGIS_GEOMETRY_FUNCTIONS = {
    'equals' : PostGISFunction('Equals'),
    'disjoint' : PostGISFunction('Disjoint'),
    'touches' : PostGISFunction('Touches'),
    'crosses' : PostGISFunction('Crosses'),
    'within' : PostGISFunction('Within'),
    'overlaps' : PostGISFunction('Overlaps'),
    'contains' : PostGISFunction('Contains'),
    'intersects' : PostGISFunction('Intersects'),
    'relate' : (PostGISRelate, basestring),
    }

# Valid distance types and substitutions
dtypes = (Decimal, Distance, float, int, long)
def get_dist_ops(operator):
    "Returns operations for both regular and spherical distances."
    return (PostGISDistance(operator), PostGISSphereDistance(operator), PostGISSpheroidDistance(operator))
DISTANCE_FUNCTIONS = {
    'distance_gt' : (get_dist_ops('>'), dtypes),
    'distance_gte' : (get_dist_ops('>='), dtypes),
    'distance_lt' : (get_dist_ops('<'), dtypes),
    'distance_lte' : (get_dist_ops('<='), dtypes),
    }

if GEOM_FUNC_PREFIX == 'ST_':
    # The ST_DWithin, ST_CoveredBy, and ST_Covers routines become available in 1.2.2+
    POSTGIS_GEOMETRY_FUNCTIONS.update(
        {'coveredby' : PostGISFunction('CoveredBy'),
         'covers' : PostGISFunction('Covers'),
         })
    DISTANCE_FUNCTIONS['dwithin'] = (PostGISFunctionParam('DWithin'), dtypes)

# Distance functions are a part of PostGIS geometry functions.
POSTGIS_GEOMETRY_FUNCTIONS.update(DISTANCE_FUNCTIONS)

# Any other lookup types that do not require a mapping.
MISC_TERMS = ['isnull']

# These are the PostGIS-customized QUERY_TERMS -- a list of the lookup types
#  allowed for geographic queries.
POSTGIS_TERMS = POSTGIS_OPERATORS.keys() # Getting the operators first
POSTGIS_TERMS += POSTGIS_GEOMETRY_FUNCTIONS.keys() # Adding on the Geometry Functions
POSTGIS_TERMS += MISC_TERMS # Adding any other miscellaneous terms (e.g., 'isnull')
POSTGIS_TERMS = dict((term, None) for term in POSTGIS_TERMS) # Making a dictionary for fast lookups

# For checking tuple parameters -- not very pretty but gets job done.
def exactly_two(val): return val == 2
def two_to_three(val): return val >= 2 and val <=3
def num_params(lookup_type, val):
    if lookup_type in DISTANCE_FUNCTIONS and lookup_type != 'dwithin': return two_to_three(val)
    else: return exactly_two(val)

#### The `get_geo_where_clause` function for PostGIS. ####
def get_geo_where_clause(table_alias, name, lookup_type, geo_annot):
    "Returns the SQL WHERE clause for use in PostGIS SQL construction."
    # Getting the quoted field as `geo_col`.
    geo_col = '%s.%s' % (qn(table_alias), qn(name))
    if lookup_type in POSTGIS_OPERATORS:
        # See if a PostGIS operator matches the lookup type.
        return POSTGIS_OPERATORS[lookup_type].as_sql(geo_col)
    elif lookup_type in POSTGIS_GEOMETRY_FUNCTIONS:
        # See if a PostGIS geometry function matches the lookup type.
        tmp = POSTGIS_GEOMETRY_FUNCTIONS[lookup_type]

        # Lookup types that are tuples take tuple arguments, e.g., 'relate' and
        # distance lookups.
        if isinstance(tmp, tuple):
            # First element of tuple is the PostGISOperation instance, and the
            # second element is either the type or a tuple of acceptable types
            # that may passed in as further parameters for the lookup type.
            op, arg_type = tmp

            # Ensuring that a tuple _value_ was passed in from the user
            if not isinstance(geo_annot.value, (tuple, list)):
                raise TypeError('Tuple required for `%s` lookup type.' % lookup_type)

            # Number of valid tuple parameters depends on the lookup type.
            nparams = len(geo_annot.value)
            if not num_params(lookup_type, nparams):
                raise ValueError('Incorrect number of parameters given for `%s` lookup type.' % lookup_type)

            # Ensuring the argument type matches what we expect.
            if not isinstance(geo_annot.value[1], arg_type):
                raise TypeError('Argument type should be %s, got %s instead.' % (arg_type, type(geo_annot.value[1])))

            # For lookup type `relate`, the op instance is not yet created (has
            # to be instantiated here to check the pattern parameter).
            if lookup_type == 'relate':
                op = op(geo_annot.value[1])
            elif lookup_type in DISTANCE_FUNCTIONS and lookup_type != 'dwithin':
                if geo_annot.geodetic:
                    # Geodetic distances are only availble from Points to PointFields.
                    if geo_annot.geom_type != 'POINT':
                        raise TypeError('PostGIS spherical operations are only valid on PointFields.')
                    if geo_annot.value[0].geom_typeid != 0:
                        raise TypeError('PostGIS geometry distance parameter is required to be of type Point.')
                    # Setting up the geodetic operation appropriately.
                    if nparams == 3 and geo_annot.value[2] == 'spheroid': op = op[2]
                    else: op = op[1]
                else:
                    op = op[0]
        else:
            op = tmp
        # Calling the `as_sql` function on the operation instance.
        return op.as_sql(geo_col)
    elif lookup_type == 'isnull':
        # Handling 'isnull' lookup type
        return "%s IS %sNULL" % (geo_col, (not geo_annot.value and 'NOT ' or ''))

    raise TypeError("Got invalid lookup_type: %s" % repr(lookup_type))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

DEPARTMENT_ASCII_CHOICES = (
    ('01', '01 - Ain'),
    ('02', '02 - Aisne'),
    ('03', '03 - Allier'),
    ('04', '04 - Alpes-de-Haute-Provence'),
    ('05', '05 - Hautes-Alpes'),
    ('06', '06 - Alpes-Maritimes'),
    ('07', '07 - Ardeche'),
    ('08', '08 - Ardennes'),
    ('09', '09 - Ariege'),
    ('10', '10 - Aube'),
    ('11', '11 - Aude'),
    ('12', '12 - Aveyron'),
    ('13', '13 - Bouches-du-Rhone'),
    ('14', '14 - Calvados'),
    ('15', '15 - Cantal'),
    ('16', '16 - Charente'),
    ('17', '17 - Charente-Maritime'),
    ('18', '18 - Cher'),
    ('19', '19 - Correze'),
    ('21', '21 - Cote-d\'Or'),
    ('22', '22 - Cotes-d\'Armor'),
    ('23', '23 - Creuse'),
    ('24', '24 - Dordogne'),
    ('25', '25 - Doubs'),
    ('26', '26 - Drome'),
    ('27', '27 - Eure'),
    ('28', '28 - Eure-et-Loire'),
    ('29', '29 - Finistere'),
    ('2A', '2A - Corse-du-Sud'),
    ('2B', '2B - Haute-Corse'),
    ('30', '30 - Gard'),
    ('31', '31 - Haute-Garonne'),
    ('32', '32 - Gers'),
    ('33', '33 - Gironde'),
    ('34', '34 - Herault'),
    ('35', '35 - Ille-et-Vilaine'),
    ('36', '36 - Indre'),
    ('37', '37 - Indre-et-Loire'),
    ('38', '38 - Isere'),
    ('39', '39 - Jura'),
    ('40', '40 - Landes'),
    ('41', '41 - Loir-et-Cher'),
    ('42', '42 - Loire'),
    ('43', '43 - Haute-Loire'),
    ('44', '44 - Loire-Atlantique'),
    ('45', '45 - Loiret'),
    ('46', '46 - Lot'),
    ('47', '47 - Lot-et-Garonne'),
    ('48', '48 - Lozere'),
    ('49', '49 - Maine-et-Loire'),
    ('50', '50 - Manche'),
    ('51', '51 - Marne'),
    ('52', '52 - Haute-Marne'),
    ('53', '53 - Mayenne'),
    ('54', '54 - Meurthe-et-Moselle'),
    ('55', '55 - Meuse'),
    ('56', '56 - Morbihan'),
    ('57', '57 - Moselle'),
    ('58', '58 - Nievre'),
    ('59', '59 - Nord'),
    ('60', '60 - Oise'),
    ('61', '61 - Orne'),
    ('62', '62 - Pas-de-Calais'),
    ('63', '63 - Puy-de-Dome'),
    ('64', '64 - Pyrenees-Atlantiques'),
    ('65', '65 - Hautes-Pyrenees'),
    ('66', '66 - Pyrenees-Orientales'),
    ('67', '67 - Bas-Rhin'),
    ('68', '68 - Haut-Rhin'),
    ('69', '69 - Rhone'),
    ('70', '70 - Haute-Saone'),
    ('71', '71 - Saone-et-Loire'),
    ('72', '72 - Sarthe'),
    ('73', '73 - Savoie'),
    ('74', '74 - Haute-Savoie'),
    ('75', '75 - Paris'),
    ('76', '76 - Seine-Maritime'),
    ('77', '77 - Seine-et-Marne'),
    ('78', '78 - Yvelines'),
    ('79', '79 - Deux-Sevres'),
    ('80', '80 - Somme'),
    ('81', '81 - Tarn'),
    ('82', '82 - Tarn-et-Garonne'),
    ('83', '83 - Var'),
    ('84', '84 - Vaucluse'),
    ('85', '85 - Vendee'),
    ('86', '86 - Vienne'),
    ('87', '87 - Haute-Vienne'),
    ('88', '88 - Vosges'),
    ('89', '89 - Yonne'),
    ('90', '90 - Territoire de Belfort'),
    ('91', '91 - Essonne'),
    ('92', '92 - Hauts-de-Seine'),
    ('93', '93 - Seine-Saint-Denis'),
    ('94', '94 - Val-de-Marne'),
    ('95', '95 - Val-d\'Oise'),
    ('2A', '2A - Corse du sud'),
    ('2B', '2B - Haute Corse'),
    ('971', '971 - Guadeloupe'),
    ('972', '972 - Martinique'),
    ('973', '973 - Guyane'),
    ('974', '974 - La Reunion'),
    ('975', '975 - Saint-Pierre-et-Miquelon'),
    ('976', '976 - Mayotte'),
    ('984', '984 - Terres Australes et Antarctiques'),
    ('986', '986 - Wallis et Futuna'),
    ('987', '987 - Polynesie Francaise'),
    ('988', '988 - Nouvelle-Caledonie'),
)
<commit_msg>Fixed #13387 -- Removed some duplicated departments in the French localflavor. Thanks to Pinaraf for the report.<commit_after># -*- coding: utf-8 -*-

DEPARTMENT_ASCII_CHOICES = (
    ('01', '01 - Ain'),
    ('02', '02 - Aisne'),
    ('03', '03 - Allier'),
    ('04', '04 - Alpes-de-Haute-Provence'),
    ('05', '05 - Hautes-Alpes'),
    ('06', '06 - Alpes-Maritimes'),
    ('07', '07 - Ardeche'),
    ('08', '08 - Ardennes'),
    ('09', '09 - Ariege'),
    ('10', '10 - Aube'),
    ('11', '11 - Aude'),
    ('12', '12 - Aveyron'),
    ('13', '13 - Bouches-du-Rhone'),
    ('14', '14 - Calvados'),
    ('15', '15 - Cantal'),
    ('16', '16 - Charente'),
    ('17', '17 - Charente-Maritime'),
    ('18', '18 - Cher'),
    ('19', '19 - Correze'),
    ('21', '21 - Cote-d\'Or'),
    ('22', '22 - Cotes-d\'Armor'),
    ('23', '23 - Creuse'),
    ('24', '24 - Dordogne'),
    ('25', '25 - Doubs'),
    ('26', '26 - Drome'),
    ('27', '27 - Eure'),
    ('28', '28 - Eure-et-Loire'),
    ('29', '29 - Finistere'),
    ('2A', '2A - Corse-du-Sud'),
    ('2B', '2B - Haute-Corse'),
    ('30', '30 - Gard'),
    ('31', '31 - Haute-Garonne'),
    ('32', '32 - Gers'),
    ('33', '33 - Gironde'),
    ('34', '34 - Herault'),
    ('35', '35 - Ille-et-Vilaine'),
    ('36', '36 - Indre'),
    ('37', '37 - Indre-et-Loire'),
    ('38', '38 - Isere'),
    ('39', '39 - Jura'),
    ('40', '40 - Landes'),
    ('41', '41 - Loir-et-Cher'),
    ('42', '42 - Loire'),
    ('43', '43 - Haute-Loire'),
    ('44', '44 - Loire-Atlantique'),
    ('45', '45 - Loiret'),
    ('46', '46 - Lot'),
    ('47', '47 - Lot-et-Garonne'),
    ('48', '48 - Lozere'),
    ('49', '49 - Maine-et-Loire'),
    ('50', '50 - Manche'),
    ('51', '51 - Marne'),
    ('52', '52 - Haute-Marne'),
    ('53', '53 - Mayenne'),
    ('54', '54 - Meurthe-et-Moselle'),
    ('55', '55 - Meuse'),
    ('56', '56 - Morbihan'),
    ('57', '57 - Moselle'),
    ('58', '58 - Nievre'),
    ('59', '59 - Nord'),
    ('60', '60 - Oise'),
    ('61', '61 - Orne'),
    ('62', '62 - Pas-de-Calais'),
    ('63', '63 - Puy-de-Dome'),
    ('64', '64 - Pyrenees-Atlantiques'),
    ('65', '65 - Hautes-Pyrenees'),
    ('66', '66 - Pyrenees-Orientales'),
    ('67', '67 - Bas-Rhin'),
    ('68', '68 - Haut-Rhin'),
    ('69', '69 - Rhone'),
    ('70', '70 - Haute-Saone'),
    ('71', '71 - Saone-et-Loire'),
    ('72', '72 - Sarthe'),
    ('73', '73 - Savoie'),
    ('74', '74 - Haute-Savoie'),
    ('75', '75 - Paris'),
    ('76', '76 - Seine-Maritime'),
    ('77', '77 - Seine-et-Marne'),
    ('78', '78 - Yvelines'),
    ('79', '79 - Deux-Sevres'),
    ('80', '80 - Somme'),
    ('81', '81 - Tarn'),
    ('82', '82 - Tarn-et-Garonne'),
    ('83', '83 - Var'),
    ('84', '84 - Vaucluse'),
    ('85', '85 - Vendee'),
    ('86', '86 - Vienne'),
    ('87', '87 - Haute-Vienne'),
    ('88', '88 - Vosges'),
    ('89', '89 - Yonne'),
    ('90', '90 - Territoire de Belfort'),
    ('91', '91 - Essonne'),
    ('92', '92 - Hauts-de-Seine'),
    ('93', '93 - Seine-Saint-Denis'),
    ('94', '94 - Val-de-Marne'),
    ('95', '95 - Val-d\'Oise'),
    ('971', '971 - Guadeloupe'),
    ('972', '972 - Martinique'),
    ('973', '973 - Guyane'),
    ('974', '974 - La Reunion'),
    ('975', '975 - Saint-Pierre-et-Miquelon'),
    ('976', '976 - Mayotte'),
    ('984', '984 - Terres Australes et Antarctiques'),
    ('986', '986 - Wallis et Futuna'),
    ('987', '987 - Polynesie Francaise'),
    ('988', '988 - Nouvelle-Caledonie'),
)
<|endoftext|>


<commit_before># Problem 1.

def product(a, b):
    """
    >>> product(3, 4)
    12
    """
    if a == 1:
        return b
    return b + product(a-1, b)

# Problem 2.

def flatten_dict(d, parent_key=None, result=None):
    if result is None:
        result = {}

    for key, value in d.iteritems():
        if isinstance(value, dict):
            parent_key = key
            flatten_dict(value, parent_key, result)
            parent_key = None
        else:
            if parent_key:
                key = '.'.join([parent_key, key])
                result[key] = value
            else:
                result[key] = value

    return result

# Problem 5.
# Refer to wget.py

# Problem 7.

if __name__ == '__main__':
    import doctest
    doctest.testmod()

<commit_msg>Update<commit_after># Problem 1.

def product(a, b):
    """
    >>> product(3, 4)
    12
    """
    if a == 1:
        return b
    return b + product(a-1, b)

# Problem 2.

def flatten_dict(d, parent_key=None, result=None):
    if result is None:
        result = {}

    for key, value in d.iteritems():
        if isinstance(value, dict):
            parent_key = key
            flatten_dict(value, parent_key, result)
            parent_key = None
        else:
            if parent_key:
                key = '.'.join([parent_key, key])
                result[key] = value
            else:
                result[key] = value

    return result

# Problem 4.

def treemap(fn, lst, result=None):
    '''
    >>> treemap(lambda x: x*x, [1, 2, [3, 4, [5]]])
    [1, 4, [9, 16, [25]]]
    '''
    if result is None:
        result = []

    for x in lst:
        if isinstance(x, list):
            result.append(treemap(fn, x, None))
        else:
            result.append(fn(x))

    return result

# Problem 5.

def tree_reverse(lst, result=None):
    '''
    >>> tree_reverse([[1, 2], [3, [4, 5]], 6])
    [6, [[5, 4], 3], [2, 1]]
    '''
    if result is None:
        result = []

    for x in lst:
        if isinstance(x, list):
            result.insert(0, tree_reverse(x, None))
        else:
            result.insert(0, x)

    return result

# Problem 6.

def json_encode(data):
    if isinstance(data, bool):
        if data:
            return "true"
        else:
            return "false"
    elif isinstance(data, (int, float)):
        return str(data)
    elif isinstance(data, str):
        return '"' + escape_string(data) + '"'
    elif isinstance(data, (tuple, list)):
        return "[" + ", ".join(json_encode(d) for d in data) + "]"
    elif isinstance(data, dict):
        return "{" + ", ".join('{key}:{value}'.format(key=json_encode(k), value=json_encode(v))
                for k, v in data.iteritems()) + "}"
    else:
        raise TypeError("%s is not JSON serializable" % repr(data))

def escape_string(s):
    """Escapes double-quote, tab and new line characters in a string."""
    s = s.replace('"', '\\"')
    s = s.replace("\t", "\\t")
    s = s.replace("\n", "\\n")
    return s

if __name__ == '__main__':
    import doctest
    doctest.testmod()

<|endoftext|>


<commit_before>from collections import defaultdict, Counter
from functools import lru_cache
from itertools import combinations

from database import db, get_or_create
from database import fast_count
import models
from sqlalchemy import and_, distinct, func, literal_column, case
from sqlalchemy import or_
from flask import current_app
from models import Mutation, Count, Site, Protein, MC3Mutation
from tqdm import tqdm

counters = {}

def counter(func, name=None):
    if not name:
        name = func.__name__
    counters[name] = func
    return lru_cache(maxsize=1)(func)

def models_counter(model):
    def counter(self):
        return self.count(model)
    counter.to_be_registered = True
    return counter

def mutations_counter(func):
    return counter(func, name='mutations_' + func.__name__)

class Statistics:

    @property
    def counters(self):
        return counters

    @staticmethod
    def get_filter_by_sources(sources):

        filters = and_(
            (
                (
                    Mutation.get_relationship(source).any()
                    if source.details_manager else
                    Mutation.get_relationship(source).has()
                )
                for source in sources

            )
        )

        return filters

    def count_by_source(self, sources):
        return Mutation.query.filter(
            self.get_filter_by_sources(sources)
        ).count()

    def get_methods(self):

        def is_method(member):
            name, value = member
            return not name.startswith('_') and callable(value)

        all_members = {name: getattr(self, name) for name in dir(self)}

        return filter(is_method, all_members.items())

    def __init__(self):

        for model in Mutation.source_specific_data:
            # dirty trick: 1KGenomes is not valid name in python
            name = 'mutations_' + model.name.replace('1', 'T')

            def muts_counter(self, model=model):
                return self.count_mutations(model)
            muts_counter.to_be_registered = True

            self.__dict__[name] = muts_counter

        for model in filter(lambda model: model.details_manager, Mutation.source_specific_data):
            name = 'mutations_' + model.name + '_annotations'

            self.__dict__[name] = models_counter(model)

        for name, method in self.get_methods():
            if hasattr(method, 'to_be_registered'):
                self.__dict__[name] = counter(method, name)

    def calc_all(self):
        for name, counter in self.counters.items():
            model, new = get_or_create(Count, name=name)
            if hasattr(counter, '__self__'):
                value = counter()
            else:
                value = counter(self)
            model.value = value
            print(name, value)
            if new:
                db.session.add(model)

    def get_all(self):

        mutation_counts = {
            counter_name[10:]: db.session.query(Count.value).filter(Count.name == counter_name).scalar() or 0
            for counter_name in self.counters.keys()
            if counter_name.startswith('mutations_')
        }

        counts = {
            counter_name: db.session.query(Count.value).filter(Count.name == counter_name).scalar() or 0
            for counter_name in self.counters.keys()
            if not counter_name.startswith('mutations_')
        }

        counts['muts'] = mutation_counts

        return counts

    @mutations_counter
    def all(self):
        """Either confirmed or not."""
        return self.count(Mutation)

    @mutations_counter
    def all_confirmed(self):
        return Mutation.query.filter_by(
            is_confirmed=True
        ).count()

    @mutations_counter
    def confirmed_in_ptm_sites(self):
        return Mutation.query.filter_by(
            is_confirmed=True,
            is_ptm_distal=True
        ).count()

    @mutations_counter
    def confirmed_with_mimp(self):
        return Mutation.query.filter(
            and_(
                self.get_filter_by_sources([models.MIMPMutation]),
                Mutation.is_confirmed,
            )
        ).count()

    # 'from_more_than_one_source' is very expensive, and it might be better
    # to disable when not necessary (it will be useful for debugging
    # purposes - so we can check if mutations count is correct)
    # @mutations_counter
    def from_more_than_one_source(self):
        """Counts mutations that have annotations in more
        than one source (eg. in both: TCGA and ClinVar).
        """

        sources = [
            model
            for model in Mutation.source_specific_data
            if model != models.MIMPMutation
        ]
        count = 0

        for i in range(2, len(sources) - 1):
            sign = 1 if i % 2 == 0 else -1
            for combination in combinations(sources, i):
                count += sign * self.count_by_source(combination)

        return count

    @staticmethod
    def count(model):
        return db.session.query(model).count()

    def count_mutations(self, mutation_class):
        if mutation_class.details_manager is not None:
            return db.session.query(Mutation).filter(
                self.get_filter_by_sources([mutation_class])
            ).count()
        else:
            return self.count(mutation_class)

    @counter
    def proteins(self):
        return self.count(models.Protein)

    genes = models_counter(models.Gene)
    kinases = models_counter(models.Kinase)
    kinase_groups = models_counter(models.KinaseGroup)
    sites = models_counter(models.Site)
    pathways = models_counter(models.Pathway)
    cancer = models_counter(models.Cancer)

    @counter
    def mappings(self):
        from database import bdb
        return len(bdb)

    @counter
    def annotations(self):
        # "number of mutation annotations
        # (all DNA>protein table + MIMP annotations)"
        return self.count(models.MIMPMutation) + self.mappings()

    @counter
    def kinases_covered(self):
        return fast_count(db.session.query(models.Kinase).filter(models.Kinase.sites.any()))

    @counter
    def kinase_groups_covered(self):
        return fast_count(db.session.query(models.KinaseGroup).filter(models.KinaseGroup.sites.any()))

    @counter
    def interactions(self):
        return (
            fast_count(db.session.query(models.Site).join(models.Kinase, models.Site.kinases)) +
            fast_count(db.session.query(models.Site).join(models.KinaseGroup, models.Site.kinase_groups))
        )

    @counter
    def proteins_covered(self):
        return (
            db.session.query(
                func.count(
                    distinct(models.Site.protein_id)
                )
            )
            .filter(
                or_(
                    models.Site.kinases.any(),
                    models.Site.kinase_groups.any()
                )
            )
            .scalar()
        )

def count_mutated_sites(site_type, model=None):
    filters = [
        Mutation.protein_id == Protein.id,
        Site.protein_id == Protein.id,
        Mutation.precomputed_is_ptm
    ]
    if site_type:
        filters.append(Site.type.like('%' + site_type + '%'))
    query = (
        db.session.query(
            func.count(distinct(case(
                [
                    (
                        (
                            Mutation.position.between(
                                Site.position - 7,
                                Site.position + 7
                            )
                        ),
                        Site.id
                    )
                ],
                else_=literal_column('NULL')
            )))
        )
        .filter(and_(*filters))
        .join(Mutation, Site.protein_id == Mutation.protein_id)
    )
    if model:
        query = query.filter(Statistics.get_filter_by_sources([model]))
    else:
        query = query.filter(Mutation.is_confirmed == True)
    return query.scalar()

def all_mutated_sites():
    mutated_sites = {}
    site_type_queries = ['']  # empty will match all sites
    site_type_queries.extend(Site.types)
    for site_type in tqdm(site_type_queries):
        mutated_sites[site_type] = count_mutated_sites(site_type)
    print('PTM sites affected by mutations - merged')
    print(mutated_sites)

def mutation_sources():
    sources = {}

    for name, source in Mutation.sources_dict.items():
        if name == 'user':
            continue
        sources[name] = Mutation.get_source_model(name)

    return sources

def source_specific_proteins_with_ptm_mutations():

    source_models = mutation_sources()
    source_models['merged'] = None

    proteins_with_ptm_muts = {}
    kinases = {}
    kinase_groups = {}
    for name, model in tqdm(source_models.items()):
        query = (
            db.session.query(distinct(Protein.id))
            .filter(Protein.has_ptm_mutations_in_dataset(model) == True)
        )
        proteins_with_ptm_muts[name] = query.count()
        kinases[name] = (
            db.session.query(distinct(models.Kinase.id))
            .join(Protein)
            .filter(Protein.has_ptm_mutations_in_dataset(model) == True)
        ).count()
        kinase_groups[name] = (
            db.session.query(distinct(models.KinaseGroup.id))
            .join(models.Kinase)
            .join(Protein)
            .filter(Protein.has_ptm_mutations_in_dataset(model) == True)
        ).count()

    print('Proteisn with PTM muts:')
    print(proteins_with_ptm_muts)
    print('Kinases with PTM muts:')
    print(kinases)
    print('Kinase groups with PTM muts:')
    print(kinase_groups)

def source_specific_nucleotide_mappings():
    from database import bdb
    from genomic_mappings import decode_csv
    from models import Mutation
    from tqdm import tqdm
    from gc import collect

    mutations = defaultdict(str)

    sources_map = {str(i): model for i, model in enumerate(mutation_sources().values())}

    print('Loading mutations from sources:')
    for i, model in tqdm(sources_map.items(), total=len(sources_map)):
        query = (
            db.session.query(Mutation.protein_id, Mutation.alt, Mutation.position)
            .filter(Statistics.get_filter_by_sources([model]))
            #.filter(Mutation.is_confirmed==True)   # if it is in source of interest, it is confirmed (I won't count MIMPs here)
            .yield_per(5000)
        )
        for mutation in tqdm(query, total=query.count()):
            mutations[str(mutation[0]) + mutation[1] + str(mutation[2])] += i

    # add merged
    i = str(len(sources_map))
    sources_map[i] = 'merged'
    print('Loading merged mutations:')

    query = (
        db.session.query(Mutation.protein_id, Mutation.alt, Mutation.position)
        .filter(Mutation.is_confirmed==True)
        .yield_per(5000)
    )
    for mutation in tqdm(query, total=query.count()):
        mutations[str(mutation[0]) + mutation[1] + str(mutation[2])] += i

    print('Mutations loaded')
    collect()

    def iterate_known_muts_sources():
        for value in tqdm(bdb.values(), total=len(bdb.db)):
            for item in map(decode_csv, value):
                sources = mutations.get(str(item['protein_id']) + item['alt'] + str(item['pos']))
                if sources:
                    yield sources

    counts = defaultdict(int)
    fields_ids = [source_id for source_id in sources_map.keys()]

    for sources in iterate_known_muts_sources():
        for field in fields_ids:
            if field in sources:
                counts[field] += 1

    for key, value in counts.items():
        print(sources_map[key], value)

def source_specific_mutated_sites():

    muts_in_ptm_sites = {}
    mimp_muts = {}
    mutated_sites = defaultdict(dict)

    for name, model in mutation_sources().items():
        count = (
            Mutation.query
            .filter_by(is_confirmed=True, is_ptm_distal=True)
            .filter(Statistics.get_filter_by_sources([model]))
            .count()
        )
        muts_in_ptm_sites[name] = count

        mimp_muts[name] = (
            Mutation.query
            .filter(
                and_(
                    Statistics.get_filter_by_sources([models.MIMPMutation, model]),
                    Mutation.is_confirmed,
                )
            ).count()
        )

        site_type_queries = ['']  # empty will match all sites
        site_type_queries.extend(Site.types)

        for site_type in tqdm(site_type_queries):
            mutated_sites[name][site_type] = count_mutated_sites(site_type, model)

    print('Mutations - in PTM sites')
    print(muts_in_ptm_sites)
    print('Mutations - with network-rewiring effect')
    print(mimp_muts)
    print('PTM sites affected by mutations - source specific')
    print(mutated_sites)

def generate_source_specific_summary_table():
    from gc import collect

    counters = [
        source_specific_proteins_with_ptm_mutations,
        all_mutated_sites,
        source_specific_mutated_sites,
        source_specific_nucleotide_mappings
    ]
    for counter in counters:
        counter()
        collect()

def hypermutated_samples(path, threshold=900):
    from helpers.parsers import iterate_tsv_gz_file
    from collections import Counter

    samples_cnt = Counter()
    muts = defaultdict(set)
    total = 0

    for line in iterate_tsv_gz_file(path):
        total += 1
        muts[','.join([line[0], '%x' % int(line[1]), '%x' % int(line[2]), line[3], line[4]])].add(line[10])

    for samples in muts.values():
        for sample in samples:
            samples_cnt[sample] += 1

    hypermutated = {}
    for sample, count in samples_cnt.most_common():
        if count > threshold:
            hypermutated[sample] = count
        else:
            break

    print('There are %s hypermutated samples.' % len(hypermutated))
    print(
        'Hypermutated samples represent %s percent of analysed mutations.'
        %
        (sum(hypermutated.values()) / total * 100)
    )
    return hypermutated

def count_mutated_potential_sites():
    count = 0
    total_length = 0
    for protein in tqdm(Protein.query, total=Protein.query.count()):
        mutated_positions = set()
        for mut in protein.confirmed_mutations:
            mutated_positions.update(range(mut.position - 7, mut.position + 7))
        total_length += protein.length

        for i in range(protein.length):
            if i in mutated_positions:
                count += 1
    print(count, total_length, count/total_length*100)

if current_app.config['LOAD_STATS']:
    stats = Statistics()
    print('Loading statistics')
    STATISTICS = stats.get_all()
else:
    print('Skipping loading statistics')
    STATISTICS = ''
<commit_msg>Add documentation for Statisics class; move generic functions out<commit_after>from collections import defaultdict, Counter
from functools import lru_cache
from itertools import combinations

from database import db, get_or_create
from database import fast_count
import models
from sqlalchemy import and_, distinct, func, literal_column, case
from sqlalchemy import or_
from flask import current_app
from models import Mutation, Count, Site, Protein, MC3Mutation
from tqdm import tqdm

counters = {}

def counter(func, name=None):
    if not name:
        name = func.__name__
    counters[name] = func
    return lru_cache(maxsize=1)(func)

def models_counter(model):
    def counter(self):
        return self.count(model)
    counter.to_be_registered = True
    return counter

def mutations_counter(func):
    return counter(func, name='mutations_' + func.__name__)

def get_methods(instance):

    def is_method(member):
        name, value = member
        return not name.startswith('_') and callable(value)

    all_members = {name: getattr(instance, name) for name in dir(instance)}

    return filter(is_method, all_members.items())

class CountStore:

    @property
    def counters(self):
        return counters

    def calc_all(self):
        for name, counter in self.counters.items():
            model, new = get_or_create(Count, name=name)
            if hasattr(counter, '__self__'):
                value = counter()
            else:
                value = counter(self)
            model.value = value
            print(name, value)
            if new:
                db.session.add(model)

    def get_all(self):

        counts = {
            counter_name: db.session.query(Count.value).filter(Count.name == counter_name).scalar() or 0
            for counter_name in self.counters.keys()
        }

        return counts

class Statistics(CountStore):
    """This module calculates, stores and retrieves counts of data in database.

    On initialization any instance of Statistics class can be used to calculate
    counts of various data entities which are hard-coded in the class methods.

        stats = Statistics()                # initialize a new instance

    If accessed directly after initialization, each counter's method will
    be executed at run time, which may cause delayed response:

        proteins_cnt = stats.proteins()     # this may take a few seconds now
        print(proteins_cnt)

    You can compute all counts at once, so those can be saved to database afterwards:

        stats.calc_all()                    # this will take several minutes
        db.session.commit()                 # save results in database

    After using 'calc_all()', the results can be obtained from database:

        counts = stats.get_all()

    You can access the pre-defined counts instantly, with no delay now.

        print(counts['muts']['ClinVar'])
        print(counts['proteins'])
    """

    def get_all(self):
        """Retrieves data counts from database in form of dict,
        where keys are model names and values are entity counts.

        Mutations counts are accessible in sub-dict called 'muts'.
        """
        all_counts = super().get_all()

        counts = {}
        mutation_counts = {}

        for counter_name, value in all_counts.items():
            if counter_name.startswith('mutations_'):
                counter_name = counter_name[10:]    # strip off "mutations_"
                mutation_counts[counter_name] = value
            else:
                counts[counter_name] = value

        counts['muts'] = mutation_counts

        return counts

    @staticmethod
    def get_filter_by_sources(sources):

        filters = and_(
            (
                (
                    Mutation.get_relationship(source).any()
                    if source.details_manager else
                    Mutation.get_relationship(source).has()
                )
                for source in sources

            )
        )

        return filters

    def count_by_source(self, sources):
        return Mutation.query.filter(
            self.get_filter_by_sources(sources)
        ).count()

    def __init__(self):

        for model in Mutation.source_specific_data:
            # dirty trick: 1KGenomes is not valid name in python
            name = 'mutations_' + model.name.replace('1', 'T')

            def muts_counter(self, model=model):
                return self.count_mutations(model)
            muts_counter.to_be_registered = True

            self.__dict__[name] = muts_counter

        for model in filter(lambda model: model.details_manager, Mutation.source_specific_data):
            name = 'mutations_' + model.name + '_annotations'

            self.__dict__[name] = models_counter(model)

        for name, method in get_methods(self):
            if hasattr(method, 'to_be_registered'):
                self.__dict__[name] = counter(method, name)

    @mutations_counter
    def all(self):
        """Either confirmed or not."""
        return self.count(Mutation)

    @mutations_counter
    def all_confirmed(self):
        return Mutation.query.filter_by(
            is_confirmed=True
        ).count()

    @mutations_counter
    def confirmed_in_ptm_sites(self):
        return Mutation.query.filter_by(
            is_confirmed=True,
            is_ptm_distal=True
        ).count()

    @mutations_counter
    def confirmed_with_mimp(self):
        return Mutation.query.filter(
            and_(
                self.get_filter_by_sources([models.MIMPMutation]),
                Mutation.is_confirmed,
            )
        ).count()

    # 'from_more_than_one_source' is very expensive, and it might be better
    # to disable when not necessary (it will be useful for debugging
    # purposes - so we can check if mutations count is correct)
    # @mutations_counter
    def from_more_than_one_source(self):
        """Counts mutations that have annotations in more
        than one source (eg. in both: TCGA and ClinVar).
        """

        sources = [
            model
            for model in Mutation.source_specific_data
            if model != models.MIMPMutation
        ]
        count = 0

        for i in range(2, len(sources) - 1):
            sign = 1 if i % 2 == 0 else -1
            for combination in combinations(sources, i):
                count += sign * self.count_by_source(combination)

        return count

    @staticmethod
    def count(model):
        return db.session.query(model).count()

    def count_mutations(self, mutation_class):
        if mutation_class.details_manager is not None:
            return db.session.query(Mutation).filter(
                self.get_filter_by_sources([mutation_class])
            ).count()
        else:
            return self.count(mutation_class)

    @counter
    def proteins(self):
        return self.count(models.Protein)

    genes = models_counter(models.Gene)
    kinases = models_counter(models.Kinase)
    kinase_groups = models_counter(models.KinaseGroup)
    sites = models_counter(models.Site)
    pathways = models_counter(models.Pathway)
    cancer = models_counter(models.Cancer)

    @counter
    def mappings(self):
        from database import bdb
        return len(bdb)

    @counter
    def annotations(self):
        # "number of mutation annotations
        # (all DNA>protein table + MIMP annotations)"
        return self.count(models.MIMPMutation) + self.mappings()

    @counter
    def kinases_covered(self):
        return fast_count(db.session.query(models.Kinase).filter(models.Kinase.sites.any()))

    @counter
    def kinase_groups_covered(self):
        return fast_count(db.session.query(models.KinaseGroup).filter(models.KinaseGroup.sites.any()))

    @counter
    def interactions(self):
        return (
            fast_count(db.session.query(models.Site).join(models.Kinase, models.Site.kinases)) +
            fast_count(db.session.query(models.Site).join(models.KinaseGroup, models.Site.kinase_groups))
        )

    @counter
    def proteins_covered(self):
        return (
            db.session.query(
                func.count(
                    distinct(models.Site.protein_id)
                )
            )
            .filter(
                or_(
                    models.Site.kinases.any(),
                    models.Site.kinase_groups.any()
                )
            )
            .scalar()
        )

def count_mutated_sites(site_type, model=None):
    filters = [
        Mutation.protein_id == Protein.id,
        Site.protein_id == Protein.id,
        Mutation.precomputed_is_ptm
    ]
    if site_type:
        filters.append(Site.type.like('%' + site_type + '%'))
    query = (
        db.session.query(
            func.count(distinct(case(
                [
                    (
                        (
                            Mutation.position.between(
                                Site.position - 7,
                                Site.position + 7
                            )
                        ),
                        Site.id
                    )
                ],
                else_=literal_column('NULL')
            )))
        )
        .filter(and_(*filters))
        .join(Mutation, Site.protein_id == Mutation.protein_id)
    )
    if model:
        query = query.filter(Statistics.get_filter_by_sources([model]))
    else:
        query = query.filter(Mutation.is_confirmed == True)
    return query.scalar()

def all_mutated_sites():
    mutated_sites = {}
    site_type_queries = ['']  # empty will match all sites
    site_type_queries.extend(Site.types)
    for site_type in tqdm(site_type_queries):
        mutated_sites[site_type] = count_mutated_sites(site_type)
    print('PTM sites affected by mutations - merged')
    print(mutated_sites)

def mutation_sources():
    sources = {}

    for name, source in Mutation.sources_dict.items():
        if name == 'user':
            continue
        sources[name] = Mutation.get_source_model(name)

    return sources

def source_specific_proteins_with_ptm_mutations():

    source_models = mutation_sources()
    source_models['merged'] = None

    proteins_with_ptm_muts = {}
    kinases = {}
    kinase_groups = {}
    for name, model in tqdm(source_models.items()):
        query = (
            db.session.query(distinct(Protein.id))
            .filter(Protein.has_ptm_mutations_in_dataset(model) == True)
        )
        proteins_with_ptm_muts[name] = query.count()
        kinases[name] = (
            db.session.query(distinct(models.Kinase.id))
            .join(Protein)
            .filter(Protein.has_ptm_mutations_in_dataset(model) == True)
        ).count()
        kinase_groups[name] = (
            db.session.query(distinct(models.KinaseGroup.id))
            .join(models.Kinase)
            .join(Protein)
            .filter(Protein.has_ptm_mutations_in_dataset(model) == True)
        ).count()

    print('Proteisn with PTM muts:')
    print(proteins_with_ptm_muts)
    print('Kinases with PTM muts:')
    print(kinases)
    print('Kinase groups with PTM muts:')
    print(kinase_groups)

def source_specific_nucleotide_mappings():
    from database import bdb
    from genomic_mappings import decode_csv
    from models import Mutation
    from tqdm import tqdm
    from gc import collect

    mutations = defaultdict(str)

    sources_map = {str(i): model for i, model in enumerate(mutation_sources().values())}

    print('Loading mutations from sources:')
    for i, model in tqdm(sources_map.items(), total=len(sources_map)):
        query = (
            db.session.query(Mutation.protein_id, Mutation.alt, Mutation.position)
            .filter(Statistics.get_filter_by_sources([model]))
            #.filter(Mutation.is_confirmed==True)   # if it is in source of interest, it is confirmed (I won't count MIMPs here)
            .yield_per(5000)
        )
        for mutation in tqdm(query, total=query.count()):
            mutations[str(mutation[0]) + mutation[1] + str(mutation[2])] += i

    # add merged
    i = str(len(sources_map))
    sources_map[i] = 'merged'
    print('Loading merged mutations:')

    query = (
        db.session.query(Mutation.protein_id, Mutation.alt, Mutation.position)
        .filter(Mutation.is_confirmed==True)
        .yield_per(5000)
    )
    for mutation in tqdm(query, total=query.count()):
        mutations[str(mutation[0]) + mutation[1] + str(mutation[2])] += i

    print('Mutations loaded')
    collect()

    def iterate_known_muts_sources():
        for value in tqdm(bdb.values(), total=len(bdb.db)):
            for item in map(decode_csv, value):
                sources = mutations.get(str(item['protein_id']) + item['alt'] + str(item['pos']))
                if sources:
                    yield sources

    counts = defaultdict(int)
    fields_ids = [source_id for source_id in sources_map.keys()]

    for sources in iterate_known_muts_sources():
        for field in fields_ids:
            if field in sources:
                counts[field] += 1

    for key, value in counts.items():
        print(sources_map[key], value)

def source_specific_mutated_sites():

    muts_in_ptm_sites = {}
    mimp_muts = {}
    mutated_sites = defaultdict(dict)

    for name, model in mutation_sources().items():
        count = (
            Mutation.query
            .filter_by(is_confirmed=True, is_ptm_distal=True)
            .filter(Statistics.get_filter_by_sources([model]))
            .count()
        )
        muts_in_ptm_sites[name] = count

        mimp_muts[name] = (
            Mutation.query
            .filter(
                and_(
                    Statistics.get_filter_by_sources([models.MIMPMutation, model]),
                    Mutation.is_confirmed,
                )
            ).count()
        )

        site_type_queries = ['']  # empty will match all sites
        site_type_queries.extend(Site.types)

        for site_type in tqdm(site_type_queries):
            mutated_sites[name][site_type] = count_mutated_sites(site_type, model)

    print('Mutations - in PTM sites')
    print(muts_in_ptm_sites)
    print('Mutations - with network-rewiring effect')
    print(mimp_muts)
    print('PTM sites affected by mutations - source specific')
    print(mutated_sites)

def generate_source_specific_summary_table():
    from gc import collect

    counters = [
        source_specific_proteins_with_ptm_mutations,
        all_mutated_sites,
        source_specific_mutated_sites,
        source_specific_nucleotide_mappings
    ]
    for counter in counters:
        counter()
        collect()

def hypermutated_samples(path, threshold=900):
    from helpers.parsers import iterate_tsv_gz_file
    from collections import Counter

    samples_cnt = Counter()
    muts = defaultdict(set)
    total = 0

    for line in iterate_tsv_gz_file(path):
        total += 1
        muts[','.join([line[0], '%x' % int(line[1]), '%x' % int(line[2]), line[3], line[4]])].add(line[10])

    for samples in muts.values():
        for sample in samples:
            samples_cnt[sample] += 1

    hypermutated = {}
    for sample, count in samples_cnt.most_common():
        if count > threshold:
            hypermutated[sample] = count
        else:
            break

    print('There are %s hypermutated samples.' % len(hypermutated))
    print(
        'Hypermutated samples represent %s percent of analysed mutations.'
        %
        (sum(hypermutated.values()) / total * 100)
    )
    return hypermutated

def count_mutated_potential_sites():
    count = 0
    total_length = 0
    for protein in tqdm(Protein.query, total=Protein.query.count()):
        mutated_positions = set()
        for mut in protein.confirmed_mutations:
            mutated_positions.update(range(mut.position - 7, mut.position + 7))
        total_length += protein.length

        for i in range(protein.length):
            if i in mutated_positions:
                count += 1
    print(count, total_length, count/total_length*100)

if current_app.config['LOAD_STATS']:
    stats = Statistics()
    print('Loading statistics')
    STATISTICS = stats.get_all()
else:
    print('Skipping loading statistics')
    STATISTICS = ''
<|endoftext|>


<commit_before>import sys
import re

from django.core.urlresolvers import RegexURLPattern, RegexURLResolver
from django.core.management.base import BaseCommand
from django.utils import simplejson
from django.utils.datastructures import SortedDict
from django.conf import settings as project_settings

from django_js_utils import settings as app_settings

RE_KWARG = re.compile(r"(\(\?P\<(.*?)\>.*?\))") #Pattern for recongnizing named parameters in urls
RE_ARG = re.compile(r"(\(.*?\))") #Pattern for recognizing unnamed url parameters

class Command(BaseCommand):
    def handle(self, *args, **options):
        """
        Create urls.js file by parsing all of the urlpatterns in the root urls.py file
        """
        js_patterns = SortedDict()
        print "Generating Javascript urls file %s" % app_settings.URLS_JS_GENERATED_FILE
        Command.handle_url_module(js_patterns, project_settings.ROOT_URLCONF)
        #output to the file
        urls_file = open(app_settings.URLS_JS_GENERATED_FILE, "w")
        urls_file.write("dutils.conf.urls = ")
        simplejson.dump(js_patterns, urls_file)
        print "Done generating Javascript urls file %s" % app_settings.URLS_JS_GENERATED_FILE

    @staticmethod
    def handle_url_module(js_patterns, module_name, prefix=""):
        """
        Load the module and output all of the patterns
        Recurse on the included modules
        """
        __import__(module_name)
        root_urls = sys.modules[module_name]
        patterns = root_urls.urlpatterns
        for pattern in patterns:
            if issubclass(pattern.__class__, RegexURLPattern):
                if pattern.name:
                    full_url = prefix + pattern.regex.pattern
                    for chr in ["^","$"]:
                        full_url = full_url.replace(chr, "")
                    #handle kwargs, args
                    kwarg_matches = RE_KWARG.findall(full_url)
                    if kwarg_matches:
                        for el in kwarg_matches:
                            #prepare the output for JS resolver
                            full_url = full_url.replace(el[0], "<%s>" % el[1])
                    #after processing all kwargs try args
                    args_matches = RE_ARG.findall(full_url)
                    if args_matches:
                        for el in args_matches:
                            full_url = full_url.replace(el, "<>")#replace by a empty parameter name
                    js_patterns[pattern.name] = "/" + full_url
            elif issubclass(pattern.__class__, RegexURLResolver):
                if pattern.urlconf_name:
                    Command.handle_url_module(js_patterns, pattern.urlconf_name, prefix=pattern.regex.pattern)<commit_msg>Fixed a bug with handling URL module imports.<commit_after>import sys
import re

from django.core.urlresolvers import RegexURLPattern, RegexURLResolver
from django.core.management.base import BaseCommand
from django.utils import simplejson
from django.utils.datastructures import SortedDict
from django.conf import settings as project_settings

from django_js_utils import settings as app_settings

RE_KWARG = re.compile(r"(\(\?P\<(.*?)\>.*?\))") #Pattern for recongnizing named parameters in urls
RE_ARG = re.compile(r"(\(.*?\))") #Pattern for recognizing unnamed url parameters

class Command(BaseCommand):
    def handle(self, *args, **options):
        """
        Create urls.js file by parsing all of the urlpatterns in the root urls.py file
        """
        js_patterns = SortedDict()
        print "Generating Javascript urls file %s" % app_settings.URLS_JS_GENERATED_FILE
        Command.handle_url_module(js_patterns, project_settings.ROOT_URLCONF)
        #output to the file
        urls_file = open(app_settings.URLS_JS_GENERATED_FILE, "w")
        urls_file.write("dutils.conf.urls = ")
        simplejson.dump(js_patterns, urls_file)
        print "Done generating Javascript urls file %s" % app_settings.URLS_JS_GENERATED_FILE

    @staticmethod
    def handle_url_module(js_patterns, module_name, prefix=""):
        """
        Load the module and output all of the patterns
        Recurse on the included modules
        """
        if isinstance(module_name, basestring):
            __import__(module_name)
            root_urls = sys.modules[module_name]
            patterns = root_urls.urlpatterns
        else:
            root_urls = module_name
            patterns = root_urls

        for pattern in patterns:
            if issubclass(pattern.__class__, RegexURLPattern):
                if pattern.name:
                    full_url = prefix + pattern.regex.pattern
                    for chr in ["^","$"]:
                        full_url = full_url.replace(chr, "")
                    #handle kwargs, args
                    kwarg_matches = RE_KWARG.findall(full_url)
                    if kwarg_matches:
                        for el in kwarg_matches:
                            #prepare the output for JS resolver
                            full_url = full_url.replace(el[0], "<%s>" % el[1])
                    #after processing all kwargs try args
                    args_matches = RE_ARG.findall(full_url)
                    if args_matches:
                        for el in args_matches:
                            full_url = full_url.replace(el, "<>")#replace by a empty parameter name
                    js_patterns[pattern.name] = "/" + full_url
            elif issubclass(pattern.__class__, RegexURLResolver):
                if pattern.urlconf_name:
                    Command.handle_url_module(js_patterns, pattern.urlconf_name, prefix=pattern.regex.pattern)
<|endoftext|>