<commit_before>import re
from decimal import Decimal

from django.contrib.gis.db.backends.base import BaseSpatialOperations
from django.contrib.gis.db.backends.util import SpatialOperation, SpatialFunction
from django.contrib.gis.db.backends.spatialite.adapter import SpatiaLiteAdapter
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Distance
from django.core.exceptions import ImproperlyConfigured
from django.db.backends.sqlite3.base import DatabaseOperations
from django.db.utils import DatabaseError
from django.utils import six

class SpatiaLiteOperator(SpatialOperation):
    "For SpatiaLite operators (e.g. `&&`, `~`)."
    def __init__(self, operator):
        super(SpatiaLiteOperator, self).__init__(operator=operator)

class SpatiaLiteFunction(SpatialFunction):
    "For SpatiaLite function calls."
    def __init__(self, function, **kwargs):
        super(SpatiaLiteFunction, self).__init__(function, **kwargs)

class SpatiaLiteFunctionParam(SpatiaLiteFunction):
    "For SpatiaLite functions that take another parameter."
    sql_template = '%(function)s(%(geo_col)s, %(geometry)s, %%s)'

class SpatiaLiteDistance(SpatiaLiteFunction):
    "For SpatiaLite distance operations."
    dist_func = 'Distance'
    sql_template = '%(function)s(%(geo_col)s, %(geometry)s) %(operator)s %%s'

    def __init__(self, operator):
        super(SpatiaLiteDistance, self).__init__(self.dist_func,
                                                 operator=operator)

class SpatiaLiteRelate(SpatiaLiteFunctionParam):
    "For SpatiaLite Relate(<geom>, <pattern>) calls."
    pattern_regex = re.compile(r'^[012TF\*]{9}$')
    def __init__(self, pattern):
        if not self.pattern_regex.match(pattern):
            raise ValueError('Invalid intersection matrix pattern "%s".' % pattern)
        super(SpatiaLiteRelate, self).__init__('Relate')

# Valid distance types and substitutions
dtypes = (Decimal, Distance, float) + six.integer_types
def get_dist_ops(operator):
    "Returns operations for regular distances; spherical distances are not currently supported."
    return (SpatiaLiteDistance(operator),)

class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations):
    compiler_module = 'django.contrib.gis.db.models.sql.compiler'
    name = 'spatialite'
    spatialite = True
    version_regex = re.compile(r'^(?P<major>\d)\.(?P<minor1>\d)\.(?P<minor2>\d+)')
    valid_aggregates = dict([(k, None) for k in ('Extent', 'Union')])

    Adapter = SpatiaLiteAdapter
    Adaptor = Adapter # Backwards-compatibility alias.

    area = 'Area'
    centroid = 'Centroid'
    contained = 'MbrWithin'
    difference = 'Difference'
    distance = 'Distance'
    envelope = 'Envelope'
    intersection = 'Intersection'
    length = 'GLength' # OpenGis defines Length, but this conflicts with an SQLite reserved keyword
    num_geom = 'NumGeometries'
    num_points = 'NumPoints'
    point_on_surface = 'PointOnSurface'
    scale = 'ScaleCoords'
    svg = 'AsSVG'
    sym_difference = 'SymDifference'
    transform = 'Transform'
    translate = 'ShiftCoords'
    union = 'GUnion' # OpenGis defines Union, but this conflicts with an SQLite reserved keyword
    unionagg = 'GUnion'

    from_text = 'GeomFromText'
    from_wkb = 'GeomFromWKB'
    select = 'AsText(%s)'

    geometry_functions = {
        'equals' : SpatiaLiteFunction('Equals'),
        'disjoint' : SpatiaLiteFunction('Disjoint'),
        'touches' : SpatiaLiteFunction('Touches'),
        'crosses' : SpatiaLiteFunction('Crosses'),
        'within' : SpatiaLiteFunction('Within'),
        'overlaps' : SpatiaLiteFunction('Overlaps'),
        'contains' : SpatiaLiteFunction('Contains'),
        'intersects' : SpatiaLiteFunction('Intersects'),
        'relate' : (SpatiaLiteRelate, six.string_types),
        # Returns true if B's bounding box completely contains A's bounding box.
        'contained' : SpatiaLiteFunction('MbrWithin'),
        # Returns true if A's bounding box completely contains B's bounding box.
        'bbcontains' : SpatiaLiteFunction('MbrContains'),
        # Returns true if A's bounding box overlaps B's bounding box.
        'bboverlaps' : SpatiaLiteFunction('MbrOverlaps'),
        # These are implemented here as synonyms for Equals
        'same_as' : SpatiaLiteFunction('Equals'),
        'exact' : SpatiaLiteFunction('Equals'),
        }

    distance_functions = {
        'distance_gt' : (get_dist_ops('>'), dtypes),
        'distance_gte' : (get_dist_ops('>='), dtypes),
        'distance_lt' : (get_dist_ops('<'), dtypes),
        'distance_lte' : (get_dist_ops('<='), dtypes),
        }
    geometry_functions.update(distance_functions)

    def __init__(self, connection):
        super(DatabaseOperations, self).__init__(connection)

        # Creating the GIS terms dictionary.
        gis_terms = ['isnull']
        gis_terms += self.geometry_functions.keys()
        self.gis_terms = dict([(term, None) for term in gis_terms])

    def confirm_spatial_components_versions(self):
        # Determine the version of the SpatiaLite library.
        try:
            vtup = self.spatialite_version_tuple()
            version = vtup[1:]
            if version < (2, 3, 0):
                raise ImproperlyConfigured('GeoDjango only supports SpatiaLite versions '
                                           '2.3.0 and above')
            self.spatial_version = version
        except ImproperlyConfigured:
            raise
        except Exception as msg:
            raise ImproperlyConfigured('Cannot determine the SpatiaLite version for the "%s" '
                                       'database (error was "%s").  Was the SpatiaLite initialization '
                                       'SQL loaded on this database?' %
                                       (self.connection.settings_dict['NAME'], msg))

        if version >= (2, 4, 0):
            # Spatialite 2.4.0-RC4 added AsGML and AsKML, however both
            # RC2 (shipped in popular Debian/Ubuntu packages) and RC4
            # report version as '2.4.0', so we fall back to feature detection
            try:
                self._get_spatialite_func("AsGML(GeomFromText('POINT(1 1)'))")
                self.gml = 'AsGML'
                self.kml = 'AsKML'
            except DatabaseError:
                # we are using < 2.4.0-RC4
                pass
        if version >= (3, 0, 0):
            self.geojson = 'AsGeoJSON'

    def check_aggregate_support(self, aggregate):
        """
        Checks if the given aggregate name is supported (that is, if it's
        in `self.valid_aggregates`).
        """
        agg_name = aggregate.__class__.__name__
        return agg_name in self.valid_aggregates

    def convert_geom(self, wkt, geo_field):
        """
        Converts geometry WKT returned from a SpatiaLite aggregate.
        """
        if wkt:
            return Geometry(wkt, geo_field.srid)
        else:
            return None

    def geo_db_type(self, f):
        """
        Returns None because geometry columnas are added via the
        `AddGeometryColumn` stored procedure on SpatiaLite.
        """
        return None

    def get_distance(self, f, value, lookup_type):
        """
        Returns the distance parameters for the given geometry field,
        lookup value, and lookup type.  SpatiaLite only supports regular
        cartesian-based queries (no spheroid/sphere calculations for point
        geometries like PostGIS).
        """
        if not value:
            return []
        value = value[0]
        if isinstance(value, Distance):
            if f.geodetic(self.connection):
                raise ValueError('SpatiaLite does not support distance queries on '
                                 'geometry fields with a geodetic coordinate system. '
                                 'Distance objects; use a numeric value of your '
                                 'distance in degrees instead.')
            else:
                dist_param = getattr(value, Distance.unit_attname(f.units_name(self.connection)))
        else:
            dist_param = value
        return [dist_param]

    def get_geom_placeholder(self, f, value):
        """
        Provides a proper substitution value for Geometries that are not in the
        SRID of the field.  Specifically, this routine will substitute in the
        Transform() and GeomFromText() function call(s).
        """
        def transform_value(value, srid):
            return not (value is None or value.srid == srid)
        if hasattr(value, 'expression'):
            if transform_value(value, f.srid):
                placeholder = '%s(%%s, %s)' % (self.transform, f.srid)
            else:
                placeholder = '%s'
            # No geometry value used for F expression, substitue in
            # the column name instead.
            return placeholder % self.get_expression_column(value)
        else:
            if transform_value(value, f.srid):
                # Adding Transform() to the SQL placeholder.
                return '%s(%s(%%s,%s), %s)' % (self.transform, self.from_text, value.srid, f.srid)
            else:
                return '%s(%%s,%s)' % (self.from_text, f.srid)

    def _get_spatialite_func(self, func):
        """
        Helper routine for calling SpatiaLite functions and returning
        their result.
        """
        cursor = self.connection._cursor()
        try:
            try:
                cursor.execute('SELECT %s' % func)
                row = cursor.fetchone()
            except:
                # Responsibility of caller to perform error handling.
                raise
        finally:
            cursor.close()
        return row[0]

    def geos_version(self):
        "Returns the version of GEOS used by SpatiaLite as a string."
        return self._get_spatialite_func('geos_version()')

    def proj4_version(self):
        "Returns the version of the PROJ.4 library used by SpatiaLite."
        return self._get_spatialite_func('proj4_version()')

    def spatialite_version(self):
        "Returns the SpatiaLite library version as a string."
        return self._get_spatialite_func('spatialite_version()')

    def spatialite_version_tuple(self):
        """
        Returns the SpatiaLite version as a tuple (version string, major,
        minor, subminor).
        """
        # Getting the SpatiaLite version.
        try:
            version = self.spatialite_version()
        except DatabaseError:
            # The `spatialite_version` function first appeared in version 2.3.1
            # of SpatiaLite, so doing a fallback test for 2.3.0 (which is
            # used by popular Debian/Ubuntu packages).
            version = None
            try:
                tmp = self._get_spatialite_func("X(GeomFromText('POINT(1 1)'))")
                if tmp == 1.0: version = '2.3.0'
            except DatabaseError:
                pass
            # If no version string defined, then just re-raise the original
            # exception.
            if version is None: raise

        m = self.version_regex.match(version)
        if m:
            major = int(m.group('major'))
            minor1 = int(m.group('minor1'))
            minor2 = int(m.group('minor2'))
        else:
            raise Exception('Could not parse SpatiaLite version string: %s' % version)

        return (version, major, minor1, minor2)

    def spatial_aggregate_sql(self, agg):
        """
        Returns the spatial aggregate SQL template and function for the
        given Aggregate instance.
        """
        agg_name = agg.__class__.__name__
        if not self.check_aggregate_support(agg):
            raise NotImplementedError('%s spatial aggregate is not implmented for this backend.' % agg_name)
        agg_name = agg_name.lower()
        if agg_name == 'union': agg_name += 'agg'
        sql_template = self.select % '%(function)s(%(field)s)'
        sql_function = getattr(self, agg_name)
        return sql_template, sql_function

    def spatial_lookup_sql(self, lvalue, lookup_type, value, field, qn):
        """
        Returns the SpatiaLite-specific SQL for the given lookup value
        [a tuple of (alias, column, db_type)], lookup type, lookup
        value, the model field, and the quoting function.
        """
        alias, col, db_type = lvalue

        # Getting the quoted field as `geo_col`.
        geo_col = '%s.%s' % (qn(alias), qn(col))

        if lookup_type in self.geometry_functions:
            # See if a SpatiaLite geometry function matches the lookup type.
            tmp = self.geometry_functions[lookup_type]

            # Lookup types that are tuples take tuple arguments, e.g., 'relate' and
            # distance lookups.
            if isinstance(tmp, tuple):
                # First element of tuple is the SpatiaLiteOperation instance, and the
                # second element is either the type or a tuple of acceptable types
                # that may passed in as further parameters for the lookup type.
                op, arg_type = tmp

                # Ensuring that a tuple _value_ was passed in from the user
                if not isinstance(value, (tuple, list)):
                    raise ValueError('Tuple required for `%s` lookup type.' % lookup_type)

                # Geometry is first element of lookup tuple.
                geom = value[0]

                # Number of valid tuple parameters depends on the lookup type.
                if len(value) != 2:
                    raise ValueError('Incorrect number of parameters given for `%s` lookup type.' % lookup_type)

                # Ensuring the argument type matches what we expect.
                if not isinstance(value[1], arg_type):
                    raise ValueError('Argument type should be %s, got %s instead.' % (arg_type, type(value[1])))

                # For lookup type `relate`, the op instance is not yet created (has
                # to be instantiated here to check the pattern parameter).
                if lookup_type == 'relate':
                    op = op(value[1])
                elif lookup_type in self.distance_functions:
                    op = op[0]
            else:
                op = tmp
                geom = value
            # Calling the `as_sql` function on the operation instance.
            return op.as_sql(geo_col, self.get_geom_placeholder(field, geom))
        elif lookup_type == 'isnull':
            # Handling 'isnull' lookup type
            return "%s IS %sNULL" % (geo_col, (not value and 'NOT ' or ''))

        raise TypeError("Got invalid lookup_type: %s" % repr(lookup_type))

    # Routines for getting the OGC-compliant models.
    def geometry_columns(self):
        from django.contrib.gis.db.backends.spatialite.models import GeometryColumns
        return GeometryColumns

    def spatial_ref_sys(self):
        from django.contrib.gis.db.backends.spatialite.models import SpatialRefSys
        return SpatialRefSys
<commit_msg>[1.5.x] Fixed geojson detection with Spatialite<commit_after>import re
from decimal import Decimal

from django.contrib.gis.db.backends.base import BaseSpatialOperations
from django.contrib.gis.db.backends.util import SpatialOperation, SpatialFunction
from django.contrib.gis.db.backends.spatialite.adapter import SpatiaLiteAdapter
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Distance
from django.core.exceptions import ImproperlyConfigured
from django.db.backends.sqlite3.base import DatabaseOperations
from django.db.utils import DatabaseError
from django.utils import six
from django.utils.functional import cached_property

class SpatiaLiteOperator(SpatialOperation):
    "For SpatiaLite operators (e.g. `&&`, `~`)."
    def __init__(self, operator):
        super(SpatiaLiteOperator, self).__init__(operator=operator)

class SpatiaLiteFunction(SpatialFunction):
    "For SpatiaLite function calls."
    def __init__(self, function, **kwargs):
        super(SpatiaLiteFunction, self).__init__(function, **kwargs)

class SpatiaLiteFunctionParam(SpatiaLiteFunction):
    "For SpatiaLite functions that take another parameter."
    sql_template = '%(function)s(%(geo_col)s, %(geometry)s, %%s)'

class SpatiaLiteDistance(SpatiaLiteFunction):
    "For SpatiaLite distance operations."
    dist_func = 'Distance'
    sql_template = '%(function)s(%(geo_col)s, %(geometry)s) %(operator)s %%s'

    def __init__(self, operator):
        super(SpatiaLiteDistance, self).__init__(self.dist_func,
                                                 operator=operator)

class SpatiaLiteRelate(SpatiaLiteFunctionParam):
    "For SpatiaLite Relate(<geom>, <pattern>) calls."
    pattern_regex = re.compile(r'^[012TF\*]{9}$')
    def __init__(self, pattern):
        if not self.pattern_regex.match(pattern):
            raise ValueError('Invalid intersection matrix pattern "%s".' % pattern)
        super(SpatiaLiteRelate, self).__init__('Relate')

# Valid distance types and substitutions
dtypes = (Decimal, Distance, float) + six.integer_types
def get_dist_ops(operator):
    "Returns operations for regular distances; spherical distances are not currently supported."
    return (SpatiaLiteDistance(operator),)

class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations):
    compiler_module = 'django.contrib.gis.db.models.sql.compiler'
    name = 'spatialite'
    spatialite = True
    version_regex = re.compile(r'^(?P<major>\d)\.(?P<minor1>\d)\.(?P<minor2>\d+)')
    valid_aggregates = dict([(k, None) for k in ('Extent', 'Union')])

    Adapter = SpatiaLiteAdapter
    Adaptor = Adapter # Backwards-compatibility alias.

    area = 'Area'
    centroid = 'Centroid'
    contained = 'MbrWithin'
    difference = 'Difference'
    distance = 'Distance'
    envelope = 'Envelope'
    intersection = 'Intersection'
    length = 'GLength' # OpenGis defines Length, but this conflicts with an SQLite reserved keyword
    num_geom = 'NumGeometries'
    num_points = 'NumPoints'
    point_on_surface = 'PointOnSurface'
    scale = 'ScaleCoords'
    svg = 'AsSVG'
    sym_difference = 'SymDifference'
    transform = 'Transform'
    translate = 'ShiftCoords'
    union = 'GUnion' # OpenGis defines Union, but this conflicts with an SQLite reserved keyword
    unionagg = 'GUnion'

    from_text = 'GeomFromText'
    from_wkb = 'GeomFromWKB'
    select = 'AsText(%s)'

    geometry_functions = {
        'equals' : SpatiaLiteFunction('Equals'),
        'disjoint' : SpatiaLiteFunction('Disjoint'),
        'touches' : SpatiaLiteFunction('Touches'),
        'crosses' : SpatiaLiteFunction('Crosses'),
        'within' : SpatiaLiteFunction('Within'),
        'overlaps' : SpatiaLiteFunction('Overlaps'),
        'contains' : SpatiaLiteFunction('Contains'),
        'intersects' : SpatiaLiteFunction('Intersects'),
        'relate' : (SpatiaLiteRelate, six.string_types),
        # Returns true if B's bounding box completely contains A's bounding box.
        'contained' : SpatiaLiteFunction('MbrWithin'),
        # Returns true if A's bounding box completely contains B's bounding box.
        'bbcontains' : SpatiaLiteFunction('MbrContains'),
        # Returns true if A's bounding box overlaps B's bounding box.
        'bboverlaps' : SpatiaLiteFunction('MbrOverlaps'),
        # These are implemented here as synonyms for Equals
        'same_as' : SpatiaLiteFunction('Equals'),
        'exact' : SpatiaLiteFunction('Equals'),
        }

    distance_functions = {
        'distance_gt' : (get_dist_ops('>'), dtypes),
        'distance_gte' : (get_dist_ops('>='), dtypes),
        'distance_lt' : (get_dist_ops('<'), dtypes),
        'distance_lte' : (get_dist_ops('<='), dtypes),
        }
    geometry_functions.update(distance_functions)

    def __init__(self, connection):
        super(DatabaseOperations, self).__init__(connection)

        # Creating the GIS terms dictionary.
        gis_terms = ['isnull']
        gis_terms += self.geometry_functions.keys()
        self.gis_terms = dict([(term, None) for term in gis_terms])

    def confirm_spatial_components_versions(self):
        # Determine the version of the SpatiaLite library.
        try:
            vtup = self.spatialite_version_tuple()
            version = vtup[1:]
            if version < (2, 3, 0):
                raise ImproperlyConfigured('GeoDjango only supports SpatiaLite versions '
                                           '2.3.0 and above')
            self.spatial_version = version
        except ImproperlyConfigured:
            raise
        except Exception as msg:
            raise ImproperlyConfigured('Cannot determine the SpatiaLite version for the "%s" '
                                       'database (error was "%s").  Was the SpatiaLite initialization '
                                       'SQL loaded on this database?' %
                                       (self.connection.settings_dict['NAME'], msg))

        if version >= (2, 4, 0):
            # Spatialite 2.4.0-RC4 added AsGML and AsKML, however both
            # RC2 (shipped in popular Debian/Ubuntu packages) and RC4
            # report version as '2.4.0', so we fall back to feature detection
            try:
                self._get_spatialite_func("AsGML(GeomFromText('POINT(1 1)'))")
                self.gml = 'AsGML'
                self.kml = 'AsKML'
            except DatabaseError:
                # we are using < 2.4.0-RC4
                pass

    @cached_property
    def geojson(self):
        return 'AsGeoJSON' if self.spatialite_version_tuple()[1:] >= (3, 0, 0) else None

    def check_aggregate_support(self, aggregate):
        """
        Checks if the given aggregate name is supported (that is, if it's
        in `self.valid_aggregates`).
        """
        agg_name = aggregate.__class__.__name__
        return agg_name in self.valid_aggregates

    def convert_geom(self, wkt, geo_field):
        """
        Converts geometry WKT returned from a SpatiaLite aggregate.
        """
        if wkt:
            return Geometry(wkt, geo_field.srid)
        else:
            return None

    def geo_db_type(self, f):
        """
        Returns None because geometry columnas are added via the
        `AddGeometryColumn` stored procedure on SpatiaLite.
        """
        return None

    def get_distance(self, f, value, lookup_type):
        """
        Returns the distance parameters for the given geometry field,
        lookup value, and lookup type.  SpatiaLite only supports regular
        cartesian-based queries (no spheroid/sphere calculations for point
        geometries like PostGIS).
        """
        if not value:
            return []
        value = value[0]
        if isinstance(value, Distance):
            if f.geodetic(self.connection):
                raise ValueError('SpatiaLite does not support distance queries on '
                                 'geometry fields with a geodetic coordinate system. '
                                 'Distance objects; use a numeric value of your '
                                 'distance in degrees instead.')
            else:
                dist_param = getattr(value, Distance.unit_attname(f.units_name(self.connection)))
        else:
            dist_param = value
        return [dist_param]

    def get_geom_placeholder(self, f, value):
        """
        Provides a proper substitution value for Geometries that are not in the
        SRID of the field.  Specifically, this routine will substitute in the
        Transform() and GeomFromText() function call(s).
        """
        def transform_value(value, srid):
            return not (value is None or value.srid == srid)
        if hasattr(value, 'expression'):
            if transform_value(value, f.srid):
                placeholder = '%s(%%s, %s)' % (self.transform, f.srid)
            else:
                placeholder = '%s'
            # No geometry value used for F expression, substitue in
            # the column name instead.
            return placeholder % self.get_expression_column(value)
        else:
            if transform_value(value, f.srid):
                # Adding Transform() to the SQL placeholder.
                return '%s(%s(%%s,%s), %s)' % (self.transform, self.from_text, value.srid, f.srid)
            else:
                return '%s(%%s,%s)' % (self.from_text, f.srid)

    def _get_spatialite_func(self, func):
        """
        Helper routine for calling SpatiaLite functions and returning
        their result.
        """
        cursor = self.connection._cursor()
        try:
            try:
                cursor.execute('SELECT %s' % func)
                row = cursor.fetchone()
            except:
                # Responsibility of caller to perform error handling.
                raise
        finally:
            cursor.close()
        return row[0]

    def geos_version(self):
        "Returns the version of GEOS used by SpatiaLite as a string."
        return self._get_spatialite_func('geos_version()')

    def proj4_version(self):
        "Returns the version of the PROJ.4 library used by SpatiaLite."
        return self._get_spatialite_func('proj4_version()')

    def spatialite_version(self):
        "Returns the SpatiaLite library version as a string."
        return self._get_spatialite_func('spatialite_version()')

    def spatialite_version_tuple(self):
        """
        Returns the SpatiaLite version as a tuple (version string, major,
        minor, subminor).
        """
        # Getting the SpatiaLite version.
        try:
            version = self.spatialite_version()
        except DatabaseError:
            # The `spatialite_version` function first appeared in version 2.3.1
            # of SpatiaLite, so doing a fallback test for 2.3.0 (which is
            # used by popular Debian/Ubuntu packages).
            version = None
            try:
                tmp = self._get_spatialite_func("X(GeomFromText('POINT(1 1)'))")
                if tmp == 1.0: version = '2.3.0'
            except DatabaseError:
                pass
            # If no version string defined, then just re-raise the original
            # exception.
            if version is None: raise

        m = self.version_regex.match(version)
        if m:
            major = int(m.group('major'))
            minor1 = int(m.group('minor1'))
            minor2 = int(m.group('minor2'))
        else:
            raise Exception('Could not parse SpatiaLite version string: %s' % version)

        return (version, major, minor1, minor2)

    def spatial_aggregate_sql(self, agg):
        """
        Returns the spatial aggregate SQL template and function for the
        given Aggregate instance.
        """
        agg_name = agg.__class__.__name__
        if not self.check_aggregate_support(agg):
            raise NotImplementedError('%s spatial aggregate is not implmented for this backend.' % agg_name)
        agg_name = agg_name.lower()
        if agg_name == 'union': agg_name += 'agg'
        sql_template = self.select % '%(function)s(%(field)s)'
        sql_function = getattr(self, agg_name)
        return sql_template, sql_function

    def spatial_lookup_sql(self, lvalue, lookup_type, value, field, qn):
        """
        Returns the SpatiaLite-specific SQL for the given lookup value
        [a tuple of (alias, column, db_type)], lookup type, lookup
        value, the model field, and the quoting function.
        """
        alias, col, db_type = lvalue

        # Getting the quoted field as `geo_col`.
        geo_col = '%s.%s' % (qn(alias), qn(col))

        if lookup_type in self.geometry_functions:
            # See if a SpatiaLite geometry function matches the lookup type.
            tmp = self.geometry_functions[lookup_type]

            # Lookup types that are tuples take tuple arguments, e.g., 'relate' and
            # distance lookups.
            if isinstance(tmp, tuple):
                # First element of tuple is the SpatiaLiteOperation instance, and the
                # second element is either the type or a tuple of acceptable types
                # that may passed in as further parameters for the lookup type.
                op, arg_type = tmp

                # Ensuring that a tuple _value_ was passed in from the user
                if not isinstance(value, (tuple, list)):
                    raise ValueError('Tuple required for `%s` lookup type.' % lookup_type)

                # Geometry is first element of lookup tuple.
                geom = value[0]

                # Number of valid tuple parameters depends on the lookup type.
                if len(value) != 2:
                    raise ValueError('Incorrect number of parameters given for `%s` lookup type.' % lookup_type)

                # Ensuring the argument type matches what we expect.
                if not isinstance(value[1], arg_type):
                    raise ValueError('Argument type should be %s, got %s instead.' % (arg_type, type(value[1])))

                # For lookup type `relate`, the op instance is not yet created (has
                # to be instantiated here to check the pattern parameter).
                if lookup_type == 'relate':
                    op = op(value[1])
                elif lookup_type in self.distance_functions:
                    op = op[0]
            else:
                op = tmp
                geom = value
            # Calling the `as_sql` function on the operation instance.
            return op.as_sql(geo_col, self.get_geom_placeholder(field, geom))
        elif lookup_type == 'isnull':
            # Handling 'isnull' lookup type
            return "%s IS %sNULL" % (geo_col, (not value and 'NOT ' or ''))

        raise TypeError("Got invalid lookup_type: %s" % repr(lookup_type))

    # Routines for getting the OGC-compliant models.
    def geometry_columns(self):
        from django.contrib.gis.db.backends.spatialite.models import GeometryColumns
        return GeometryColumns

    def spatial_ref_sys(self):
        from django.contrib.gis.db.backends.spatialite.models import SpatialRefSys
        return SpatialRefSys
<|endoftext|>


<commit_before># Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from tempest.api.object_storage import base
from tempest.common.utils import data_utils
from tempest import config
from tempest.lib import decorators
from tempest.lib import exceptions as lib_exc
from tempest import test

CONF = config.CONF

class AccountQuotasNegativeTest(base.BaseObjectTest):

    credentials = [['operator', CONF.object_storage.operator_role],
                   ['reseller', CONF.object_storage.reseller_admin_role]]

    @classmethod
    def setup_credentials(cls):
        super(AccountQuotasNegativeTest, cls).setup_credentials()
        cls.os = cls.os_roles_operator
        cls.os_reselleradmin = cls.os_roles_reseller

    @classmethod
    def resource_setup(cls):
        super(AccountQuotasNegativeTest, cls).resource_setup()
        cls.container_name = cls.create_container()

        # Retrieve a ResellerAdmin auth data and use it to set a quota
        # on the client's account
        cls.reselleradmin_auth_data = \
            cls.os_reselleradmin.auth_provider.auth_data

    def setUp(self):
        super(AccountQuotasNegativeTest, self).setUp()
        # Set the reselleradmin auth in headers for next account_client
        # request
        self.account_client.auth_provider.set_alt_auth_data(
            request_part='headers',
            auth_data=self.reselleradmin_auth_data
        )
        # Set a quota of 20 bytes on the user's account before each test
        headers = {"X-Account-Meta-Quota-Bytes": "20"}

        self.os.account_client.request("POST", url="", headers=headers,
                                       body="")

    def tearDown(self):
        # Set the reselleradmin auth in headers for next account_client
        # request
        self.account_client.auth_provider.set_alt_auth_data(
            request_part='headers',
            auth_data=self.reselleradmin_auth_data
        )
        # remove the quota from the container
        headers = {"X-Remove-Account-Meta-Quota-Bytes": "x"}

        self.os.account_client.request("POST", url="", headers=headers,
                                       body="")
        super(AccountQuotasNegativeTest, self).tearDown()

    @classmethod
    def resource_cleanup(cls):
        cls.delete_containers()
        super(AccountQuotasNegativeTest, cls).resource_cleanup()

    @test.attr(type=["negative"])
    @test.idempotent_id('d1dc5076-555e-4e6d-9697-28f1fe976324')
    @test.requires_ext(extension='account_quotas', service='object')
    def test_user_modify_quota(self):
        """Test that a user cannot modify or remove a quota on its account."""

        # Not able to remove quota
        self.assertRaises(lib_exc.Forbidden,
                          self.account_client.create_account_metadata,
                          {"Quota-Bytes": ""})

        # Not able to modify quota
        self.assertRaises(lib_exc.Forbidden,
                          self.account_client.create_account_metadata,
                          {"Quota-Bytes": "100"})

    @test.attr(type=["negative"])
    @decorators.skip_because(bug="1310597")
    @test.idempotent_id('cf9e21f5-3aa4-41b1-9462-28ac550d8d3f')
    @test.requires_ext(extension='account_quotas', service='object')
    def test_upload_large_object(self):
        object_name = data_utils.rand_name(name="TestObject")
        data = data_utils.arbitrary_string(30)
        self.assertRaises(lib_exc.OverLimit,
                          self.object_client.create_object,
                          self.container_name, object_name, data)
<commit_msg>Remove the test_upload_large_objects negative test<commit_after># Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from tempest.api.object_storage import base
from tempest import config
from tempest.lib import exceptions as lib_exc
from tempest import test

CONF = config.CONF

class AccountQuotasNegativeTest(base.BaseObjectTest):

    credentials = [['operator', CONF.object_storage.operator_role],
                   ['reseller', CONF.object_storage.reseller_admin_role]]

    @classmethod
    def setup_credentials(cls):
        super(AccountQuotasNegativeTest, cls).setup_credentials()
        cls.os = cls.os_roles_operator
        cls.os_reselleradmin = cls.os_roles_reseller

    @classmethod
    def resource_setup(cls):
        super(AccountQuotasNegativeTest, cls).resource_setup()
        cls.container_name = cls.create_container()

        # Retrieve a ResellerAdmin auth data and use it to set a quota
        # on the client's account
        cls.reselleradmin_auth_data = \
            cls.os_reselleradmin.auth_provider.auth_data

    def setUp(self):
        super(AccountQuotasNegativeTest, self).setUp()
        # Set the reselleradmin auth in headers for next account_client
        # request
        self.account_client.auth_provider.set_alt_auth_data(
            request_part='headers',
            auth_data=self.reselleradmin_auth_data
        )
        # Set a quota of 20 bytes on the user's account before each test
        headers = {"X-Account-Meta-Quota-Bytes": "20"}

        self.os.account_client.request("POST", url="", headers=headers,
                                       body="")

    def tearDown(self):
        # Set the reselleradmin auth in headers for next account_client
        # request
        self.account_client.auth_provider.set_alt_auth_data(
            request_part='headers',
            auth_data=self.reselleradmin_auth_data
        )
        # remove the quota from the container
        headers = {"X-Remove-Account-Meta-Quota-Bytes": "x"}

        self.os.account_client.request("POST", url="", headers=headers,
                                       body="")
        super(AccountQuotasNegativeTest, self).tearDown()

    @classmethod
    def resource_cleanup(cls):
        cls.delete_containers()
        super(AccountQuotasNegativeTest, cls).resource_cleanup()

    @test.attr(type=["negative"])
    @test.idempotent_id('d1dc5076-555e-4e6d-9697-28f1fe976324')
    @test.requires_ext(extension='account_quotas', service='object')
    def test_user_modify_quota(self):
        """Test that a user cannot modify or remove a quota on its account."""

        # Not able to remove quota
        self.assertRaises(lib_exc.Forbidden,
                          self.account_client.create_account_metadata,
                          {"Quota-Bytes": ""})

        # Not able to modify quota
        self.assertRaises(lib_exc.Forbidden,
                          self.account_client.create_account_metadata,
                          {"Quota-Bytes": "100"})
<|endoftext|>


<commit_before># !/usr/bin/python
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import re
import time
import json
import urllib2
import sys
import socket
import types
import httplib
import logging
import threading

# load six
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '', 'lib/six'))
import six

# load kafka-python
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '', 'lib/kafka-python'))
from kafka import KafkaClient, SimpleProducer, SimpleConsumer

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s',
                    datefmt='%m-%d %H:%M')

class Helper:
    def __init__(self):
        pass

    @staticmethod
    def load_config(config_file="config.json"):
        """

        :param config_file:
        :return:
        """
        # read the self-defined filters
        script_dir = os.path.dirname(__file__)
        rel_path = "./" + config_file
        abs_file_path = os.path.join(script_dir, rel_path)
        if not os.path.isfile(abs_file_path):
            logging.error(abs_file_path + " doesn't exist, please rename config-sample.json to config.json")
            exit(1)
        f = open(abs_file_path, 'r')
        json_file = f.read()
        f.close()
        config = json.loads(json_file)
        return config

    @staticmethod
    def is_number(str):
        """

        :param str:
        :return:
        """
        try:
            if str == None or isinstance(str, (bool)):
                return False
            float(str)
            return True
        except:
            return False

    @staticmethod
    def http_get(host, port, https=False, path=None):
        """
        Read url by GET method

        :param path:
        :param url:
        :param https:
        :return:
        """
        url = ":".join([host, str(port)])
        result = None
        response = None
        try:
            if https:
                logging.info("Reading https://" + str(url) + path)
                c = httplib.HTTPSConnection(url, timeout=57)
                c.request("GET", path)
                response = c.getresponse()
            else:
                logging.info("Reading http://" + str(url) + path)
                response = urllib2.urlopen("http://" + str(url) + path, timeout=57)
            logging.debug("Got response")
            result = response.read()
        finally:
            if response is not None:
                response.close()
            return result

class JmxReader(object):
    def __init__(self, host, port, https=False):
        self.host = host
        self.port = port
        self.https = https
        self.jmx_json = None
        self.jmx_beans = None
        self.jmx_raw = None

    def open(self):
        """
        :return: JmxReader
        """
        self.read_raw()
        self.set_raw(self.jmx_raw)
        return self

    def read_raw(self):
        """
        transfer the json string into dict
        :param host:
        :param port:
        :param https:
        :return: text
        """
        self.jmx_raw = Helper.http_get(self.host, self.port, self.https, "/jmx?anonymous=true")
        if self.jmx_raw is None:
            raise Exception("Response from " + url + " is None")
        return self

    def set_raw(self, text):
        self.jmx_json = json.loads(text)
        self.jmx_beans = self.jmx_json[u'beans']
        self.jmx_raw = text
        return self

    def get_jmx_beans(self):
        return self.jmx_beans

    def get_jmx_bean_by_name(self, name):
        for bean in self.jmx_beans:
            if bean.has_key("name") and bean["name"] == name:
                return bean

class YarnWSReader:
    def __init__(self, host, port, https=False):
        self.host = host
        self.port = port
        self.https = https

    def read_cluster_info(self):
        cluster_info = Helper.http_get(self.host, self.port, self.https, "/ws/v1/cluster/info")
        logging.debug(cluster_info)
        return json.loads(cluster_info)

class MetricSender(object):
    def __init__(self, config):
        pass

    def open(self):
        pass

    def send(self, msg):
        raise Exception("should be overrode")

    def close(self):
        pass

class KafkaMetricSender(MetricSender):
    def __init__(self, config):
        super(KafkaMetricSender, self).__init__(config)
        kafka_config = config["output"]["kafka"]
        # default topic
        self.topic = kafka_config["topic"].encode('utf-8')
        # producer
        self.broker_list = kafka_config["brokerList"]
        self.kafka_client = None
        self.kafka_producer = None

    def open(self):
        self.kafka_client = KafkaClient(self.broker_list, timeout=59)
        self.kafka_producer = SimpleProducer(self.kafka_client, batch_send=True, batch_send_every_n=500,
                                             batch_send_every_t=30)

    def send(self, msg):
        self.kafka_producer.send_messages(self.topic, json.dumps(msg))

    def close(self):
        if self.kafka_producer is not None:
            self.kafka_producer.stop()
        if self.kafka_client is not None:
            self.kafka_client.close()

class MetricCollector(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.config = Helper.load_config()
        self.sender = KafkaMetricSender(self.config)
        self.fqdn = socket.getfqdn()
        self.init(self.config)

    def init(self, config):
        pass

    def start(self):
        try:
            self.sender.open()
            self.run()
        finally:
            self.sender.close()

    def collect(self, msg):
        if not msg.has_key("timestamp"):
            msg["timestamp"] = int(round(time.time() * 1000))
        if msg.has_key("value"):
            msg["value"] = float(str(msg["value"]))
        if not msg.has_key("host") or len(msg["host"]) == 0:
            msg["host"] = self.fqdn
        if not msg.has_key("site"):
            msg["site"] = self.config["env"]["site"]

        self.sender.send(msg)

    def run(self):
        raise Exception("`run` method should be overrode by sub-class before being called")

class Runner(object):
    @staticmethod
    def run(*threads):
        """
        Execute concurrently

        :param threads:
        :return:
        """
        for thread in threads:
            thread.start()

class JmxMetricCollector(MetricCollector):
    selected_domain = None
    component = None
    https = False
    port = None
    listeners = []

    def init(self, config):
        if config["input"].has_key("host"):
            self.host = config["input"]["host"]
        else:
            self.host = self.fqdn
        self.port = config["input"]["port"]
        self.https = config["input"]["https"]
        self.component = config["input"]["component"]
        self.selected_domain = [s.encode('utf-8') for s in config[u'filter'].get('monitoring.group.selected')]
        self.listeners = []

    def register(self, *listeners):
        """
        :param listeners: type of HadoopJmxListener
        :return:
        """
        for listener in listeners:
            listener.init(self)
            self.listeners.append(listener)

    def run(self):
        try:
            beans = JmxReader(self.host, self.port, self.https).open().get_jmx_beans()
            self.on_beans(beans)
        except Exception as e:
            logging.exception("Failed to read jmx for " + self.host)

    def filter_bean(self, bean, mbean_domain):
        return mbean_domain in self.selected_domain

    def on_beans(self, beans):
        for bean in beans:
            self.on_bean(bean)

    def on_bean(self, bean):
        # mbean is of the form "domain:key=value,...,foo=bar"
        mbean = bean[u'name']
        mbean_domain, mbean_attribute = mbean.rstrip().split(":", 1)
        mbean_domain = mbean_domain.lower()

        if not self.filter_bean(bean, mbean_domain):
            return

        context = bean.get("tag.Context", "")
        metric_prefix_name = self.__build_metric_prefix(mbean_attribute, context)

        # print kafka_dict
        for key, value in bean.iteritems():
            self.on_bean_kv(metric_prefix_name, key, value)

        for listener in self.listeners:
            listener.on_bean(bean.copy())

    def on_bean_kv(self, prefix, key, value):
        # Skip Tags
        if re.match(r'tag.*', key):
            return
        metric_name = (prefix + '.' + key).lower()
        self.on_metric({
            "metric": metric_name,
            "value": value
        })

    def on_metric(self, metric):
        metric["component"] = self.component

        if Helper.is_number(metric["value"]):
            self.collect(metric)

        for listener in self.listeners:
            listener.on_metric(metric.copy())

    def __build_metric_prefix(self, mbean_attribute, context):
        mbean_list = list(prop.split("=", 1) for prop in mbean_attribute.split(","))
        if context == "":
            metric_prefix_name = '.'.join([i[1] for i in mbean_list])
        else:
            name_index = [i[0] for i in mbean_list].index('name')
            mbean_list[name_index][1] = context
            metric_prefix_name = '.'.join([i[1] for i in mbean_list])
        return ("hadoop." + metric_prefix_name).replace(" ", "").lower()

class JmxMetricListener:
    def init(self, collector):
        self.collector = collector

    def on_bean(self, bean):
        pass

    def on_metric(self, metric):
        pass<commit_msg>[EAGLE-356] Fix Authentication problem to query resource manager web service<commit_after># !/usr/bin/python
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import re
import time
import json
import urllib2
import sys
import socket
import types
import httplib
import logging
import threading

# load six
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '', 'lib/six'))
import six

# load kafka-python
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '', 'lib/kafka-python'))
from kafka import KafkaClient, SimpleProducer, SimpleConsumer

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s',
                    datefmt='%m-%d %H:%M')

class Helper:
    def __init__(self):
        pass

    @staticmethod
    def load_config(config_file="config.json"):
        """

        :param config_file:
        :return:
        """
        # read the self-defined filters
        script_dir = os.path.dirname(__file__)
        rel_path = "./" + config_file
        abs_file_path = os.path.join(script_dir, rel_path)
        if not os.path.isfile(abs_file_path):
            logging.error(abs_file_path + " doesn't exist, please rename config-sample.json to config.json")
            exit(1)
        f = open(abs_file_path, 'r')
        json_file = f.read()
        f.close()
        config = json.loads(json_file)
        return config

    @staticmethod
    def is_number(str):
        """

        :param str:
        :return:
        """
        try:
            if str == None or isinstance(str, (bool)):
                return False
            float(str)
            return True
        except:
            return False

    @staticmethod
    def http_get(host, port, https=False, path=None):
        """
        Read url by GET method

        :param path:
        :param url:
        :param https:
        :return:
        """
        url = ":".join([host, str(port)])
        result = None
        response = None
        try:
            if https:
                logging.info("Reading https://" + str(url) + path)
                c = httplib.HTTPSConnection(url, timeout=57)
                c.request("GET", path)
                response = c.getresponse()
            else:
                logging.info("Reading http://" + str(url) + path)
                response = urllib2.urlopen("http://" + str(url) + path, timeout=57)
            logging.debug("Got response")
            result = response.read()
        finally:
            if response is not None:
                response.close()
            return result

class JmxReader(object):
    def __init__(self, host, port, https=False):
        self.host = host
        self.port = port
        self.https = https
        self.jmx_json = None
        self.jmx_beans = None
        self.jmx_raw = None

    def open(self):
        """
        :return: JmxReader
        """
        self.read_raw()
        self.set_raw(self.jmx_raw)
        return self

    def read_raw(self):
        """
        transfer the json string into dict
        :param host:
        :param port:
        :param https:
        :return: text
        """
        self.jmx_raw = Helper.http_get(self.host, self.port, self.https, "/jmx?anonymous=true")
        if self.jmx_raw is None:
            raise Exception("Response from " + url + " is None")
        return self

    def set_raw(self, text):
        self.jmx_json = json.loads(text)
        self.jmx_beans = self.jmx_json[u'beans']
        self.jmx_raw = text
        return self

    def get_jmx_beans(self):
        return self.jmx_beans

    def get_jmx_bean_by_name(self, name):
        for bean in self.jmx_beans:
            if bean.has_key("name") and bean["name"] == name:
                return bean

class YarnWSReader:
    def __init__(self, host, port, https=False):
        self.host = host
        self.port = port
        self.https = https

    def read_cluster_info(self):
        cluster_info = Helper.http_get(self.host, self.port, self.https, "/ws/v1/cluster/info?anonymous=true")
        logging.debug(cluster_info)
        return json.loads(cluster_info)

class MetricSender(object):
    def __init__(self, config):
        pass

    def open(self):
        pass

    def send(self, msg):
        raise Exception("should be overrode")

    def close(self):
        pass

class KafkaMetricSender(MetricSender):
    def __init__(self, config):
        super(KafkaMetricSender, self).__init__(config)
        kafka_config = config["output"]["kafka"]
        # default topic
        self.topic = kafka_config["topic"].encode('utf-8')
        # producer
        self.broker_list = kafka_config["brokerList"]
        self.kafka_client = None
        self.kafka_producer = None

    def open(self):
        self.kafka_client = KafkaClient(self.broker_list, timeout=59)
        self.kafka_producer = SimpleProducer(self.kafka_client, batch_send=True, batch_send_every_n=500,
                                             batch_send_every_t=30)

    def send(self, msg):
        self.kafka_producer.send_messages(self.topic, json.dumps(msg))

    def close(self):
        if self.kafka_producer is not None:
            self.kafka_producer.stop()
        if self.kafka_client is not None:
            self.kafka_client.close()

class MetricCollector(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.config = Helper.load_config()
        self.sender = KafkaMetricSender(self.config)
        self.fqdn = socket.getfqdn()
        self.init(self.config)

    def init(self, config):
        pass

    def start(self):
        try:
            self.sender.open()
            self.run()
        finally:
            self.sender.close()

    def collect(self, msg):
        if not msg.has_key("timestamp"):
            msg["timestamp"] = int(round(time.time() * 1000))
        if msg.has_key("value"):
            msg["value"] = float(str(msg["value"]))
        if not msg.has_key("host") or len(msg["host"]) == 0:
            msg["host"] = self.fqdn
        if not msg.has_key("site"):
            msg["site"] = self.config["env"]["site"]

        self.sender.send(msg)

    def run(self):
        raise Exception("`run` method should be overrode by sub-class before being called")

class Runner(object):
    @staticmethod
    def run(*threads):
        """
        Execute concurrently

        :param threads:
        :return:
        """
        for thread in threads:
            thread.start()

class JmxMetricCollector(MetricCollector):
    selected_domain = None
    component = None
    https = False
    port = None
    listeners = []

    def init(self, config):
        if config["input"].has_key("host"):
            self.host = config["input"]["host"]
        else:
            self.host = self.fqdn
        self.port = config["input"]["port"]
        self.https = config["input"]["https"]
        self.component = config["input"]["component"]
        self.selected_domain = [s.encode('utf-8') for s in config[u'filter'].get('monitoring.group.selected')]
        self.listeners = []

    def register(self, *listeners):
        """
        :param listeners: type of HadoopJmxListener
        :return:
        """
        for listener in listeners:
            listener.init(self)
            self.listeners.append(listener)

    def run(self):
        try:
            beans = JmxReader(self.host, self.port, self.https).open().get_jmx_beans()
            self.on_beans(beans)
        except Exception as e:
            logging.exception("Failed to read jmx for " + self.host)

    def filter_bean(self, bean, mbean_domain):
        return mbean_domain in self.selected_domain

    def on_beans(self, beans):
        for bean in beans:
            self.on_bean(bean)

    def on_bean(self, bean):
        # mbean is of the form "domain:key=value,...,foo=bar"
        mbean = bean[u'name']
        mbean_domain, mbean_attribute = mbean.rstrip().split(":", 1)
        mbean_domain = mbean_domain.lower()

        if not self.filter_bean(bean, mbean_domain):
            return

        context = bean.get("tag.Context", "")
        metric_prefix_name = self.__build_metric_prefix(mbean_attribute, context)

        # print kafka_dict
        for key, value in bean.iteritems():
            self.on_bean_kv(metric_prefix_name, key, value)

        for listener in self.listeners:
            listener.on_bean(bean.copy())

    def on_bean_kv(self, prefix, key, value):
        # Skip Tags
        if re.match(r'tag.*', key):
            return
        metric_name = (prefix + '.' + key).lower()
        self.on_metric({
            "metric": metric_name,
            "value": value
        })

    def on_metric(self, metric):
        metric["component"] = self.component

        if Helper.is_number(metric["value"]):
            self.collect(metric)

        for listener in self.listeners:
            listener.on_metric(metric.copy())

    def __build_metric_prefix(self, mbean_attribute, context):
        mbean_list = list(prop.split("=", 1) for prop in mbean_attribute.split(","))
        if context == "":
            metric_prefix_name = '.'.join([i[1] for i in mbean_list])
        else:
            name_index = [i[0] for i in mbean_list].index('name')
            mbean_list[name_index][1] = context
            metric_prefix_name = '.'.join([i[1] for i in mbean_list])
        return ("hadoop." + metric_prefix_name).replace(" ", "").lower()

class JmxMetricListener:
    def init(self, collector):
        self.collector = collector

    def on_bean(self, bean):
        pass

    def on_metric(self, metric):
        pass
<|endoftext|>


<commit_before># Copyright 2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.!
r"""Ops for SentencePiece Encoding/Decoding."""

# TODO(taku):  Implements n-best output

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from distutils.version import LooseVersion

import warnings
import glob
import re
import os
import tensorflow as tf

so_base = os.path.join(
    os.path.dirname(__file__), '_sentencepiece_processor_ops.so')
so_file = so_base + '.' + tf.__version__

if not hasattr(tf, 'no_gradient'):
  tf.no_gradient = tf.NotDifferentiable

if not os.path.exists(so_file):
  versions = [
      re.search('so.([0-9]+\.[0-9\.]+.*)$', os.path.basename(n)).group(0)
      for n in glob.glob(so_base + '.*')
  ]
  latest = sorted(versions, key=LooseVersion)[-1]
  warnings.warn('No so file is found for [%s] from [%s]' %
                (tf.__version__, ', '.join(versions)))
  warnings.warn('use the latest version %s' % (latest))
  so_file = so_base + '.' + latest

_gen_sentencepiece_processor_op = tf.load_op_library(so_file)

def piece_size(model_file=None, model_proto=None, name=None):
  """Returns the piece size (vocabulary size).

  Args:
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A scalar representing the vocabulary size.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_get_piece_size(
      model_file=model_file, model_proto=model_proto, name=name)

def piece_to_id(input, model_file=None, model_proto=None, name=None):
  """Converts piece into vocabulary id.

  Args:
    input: An arbitrary tensor of string.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of int32 with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_piece_to_id(
      input, model_file=model_file, model_proto=model_proto, name=name)

def id_to_piece(input, model_file=None, model_proto=None, name=None):
  """Converts vocabulary id into piece.

  Args:
    input: An arbitrary tensor of int32.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of string with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_id_to_piece(
      input, model_file=model_file, model_proto=model_proto, name=name)

def is_unknown(input, model_file=None, model_proto=None, name=None):
  """Returns true if input id is unknown piece.

  Args:
    input: An arbitrary tensor of int32.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of bool with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_get_piece_type(
      input,
      model_file=model_file,
      model_proto=model_proto,
      name=name,
      piece_type=0)

def is_control(input, model_file=None, model_proto=None, name=None):
  """Returns true if input id is control piece.

  Args:
    input: An arbitrary tensor of int32.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of bool with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_get_piece_type(
      input,
      model_file=model_file,
      model_proto=model_proto,
      name=name,
      piece_type=1)

def is_unused(input, model_file=None, model_proto=None, name=None):
  """Returns true if input id is unused piece.

  Args:
    input: An arbitrary tensor of int32.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of bool with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_get_piece_type(
      input,
      model_file=model_file,
      model_proto=model_proto,
      name=name,
      piece_type=2)

def encode_dense(input_sentences,
                 nbest_size=0,
                 alpha=1.0,
                 model_file=None,
                 model_proto=None,
                 reverse=False,
                 add_bos=False,
                 add_eos=False,
                 out_type=tf.int32,
                 name=None):
  """Encodes sentences into pieces in dense tensor format.

  Args:
    input_sentences: A 1D string tensor of arbitrary size holding the raw text
      of input sentences.
    nbest_size: A scalar or 1D tensor for sampling.
                nbest_size = {0,1}: No sampling is performed.
                nbest_size > 1: samples from the nbest_size results.
                nbest_size < 0: assuming that nbest_size is infinite and samples
                  from the all hypothesis (lattice) using
                  forward-filtering-and-backward-sampling algorithm.
    alpha: A scalar or 1D tensor for a smoothing parameter. Inverse temperature
      for probability rescaling.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    reverse: Reverses the tokenized sequence (Default = false)
    add_bos: Add <s> to the result (Default = false)
    add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
      reversing (if enabled).
    out_type: output type. tf.int32 or tf.string (Default = tf.int32) Setting
      tf.int32 directly encodes the string into an id sequence.
    name: The name argument that is passed to the op function.

  Returns:
    pieces: A dense 2D tensor representing the tokenized sentences.
    sequence_length: A 1D tensor representing the length of pieces.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_encode_dense(
      input_sentences,
      nbest_size=nbest_size,
      alpha=alpha,
      model_file=model_file,
      model_proto=model_proto,
      reverse=reverse,
      add_bos=add_bos,
      add_eos=add_eos,
      out_type=out_type,
      name=name)

def encode_sparse(input_sentences,
                  nbest_size=0,
                  alpha=1.0,
                  model_file=None,
                  model_proto=None,
                  reverse=False,
                  add_bos=False,
                  add_eos=False,
                  out_type=tf.int32,
                  name=None):
  """Encodes sentences into pieces in sparse tensor format.

  Args:
    input_sentences: A 1D string tensor of arbitrary size holding the raw text
      of input sentences.
    nbest_size: A scalar or 1D tensor for sampling.
                nbest_size = {0,1}: No sampling is performed.
                nbest_size > 1: samples from the nbest_size results.
                nbest_size < 0: assuming that nbest_size is infinite and samples
                  from the all hypothesis (lattice) using
                  forward-filtering-and-backward-sampling algorithm.
    alpha: A scalar or 1D tensor for a smoothing parameter. Inverse temperature
      for probability rescaling.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    reverse: Reverses the tokenized sequence (Default = false)
    add_bos: Add <s> to the result (Default = false)
    add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
      reversing (if enabled).
    out_type: output type. tf.int32 or tf.string (Default = tf.int32) Setting
      tf.int32 directly encodes the string into an id sequence.
    name: The name argument that is passed to the op function.

  Returns:
    pieces: A sparse 2D tensor representing the tokenized sentences.
  """

  indices, values, dense_shape = (
      _gen_sentencepiece_processor_op.sentencepiece_encode_sparse(
          input_sentences,
          nbest_size=nbest_size,
          alpha=alpha,
          model_file=model_file,
          model_proto=model_proto,
          reverse=reverse,
          add_bos=add_bos,
          add_eos=add_eos,
          out_type=out_type,
          name=name))
  return tf.SparseTensor(indices, values, dense_shape)

def decode(pieces,
           sequence_length,
           model_file=None,
           model_proto=None,
           reverse=False,
           name=None):
  """Decode pieces into postprocessed text.

  Args:
    pieces: A 2D int32 or string tensor [batch_size x max_length] of encoded
      sequences.
    sequence_length: A 1D int32 tensor [batch_size] representing the length of
      pieces.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    reverse: Reverses the tokenized sequence (Default = false)
    name: The name argument that is passed to the op function.

  Returns:
    text: A 1D string tensor of decoded string.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_decode(
      pieces,
      sequence_length,
      model_file=model_file,
      model_proto=model_proto,
      reverse=reverse,
      name=name)

# Adds an alias for encode_dense. Accepts the `encode` function.
encode = encode_dense
sparse_encode = encode_sparse
dense_encode = encode_dense

tf.no_gradient('SentencepieceGetPieceSize')
tf.no_gradient('SentencepieceIdToPiece')
tf.no_gradient('SentencepiecePieceToId')
tf.no_gradient('SentencepieceGetPieceType')
tf.no_gradient('SentencepieceEncodeDense')
tf.no_gradient('SentencepieceEncodeSparse')
tf.no_gradient('SentencepieceDecode')
<commit_msg>Fix library selection logic<commit_after># Copyright 2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.!
r"""Ops for SentencePiece Encoding/Decoding."""

# TODO(taku):  Implements n-best output

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from distutils.version import LooseVersion

import warnings
import glob
import re
import os
import tensorflow as tf

so_base = os.path.join(
    os.path.dirname(__file__), '_sentencepiece_processor_ops.so')
so_file = so_base + '.' + tf.__version__

if not hasattr(tf, 'no_gradient'):
  tf.no_gradient = tf.NotDifferentiable

if not os.path.exists(so_file):
  versions = [
      re.search('so.([0-9]+\.[0-9\.]+.*)$', os.path.basename(n)).group(1)
      for n in glob.glob(so_base + '.*')
  ]
  latest = sorted(versions, key=LooseVersion)[-1]
  warnings.warn('No so file is found for [%s] from [%s]' %
                (tf.__version__, ', '.join(versions)))
  warnings.warn('use the latest version %s' % (latest))
  so_file = so_base + '.' + latest

_gen_sentencepiece_processor_op = tf.load_op_library(so_file)

def piece_size(model_file=None, model_proto=None, name=None):
  """Returns the piece size (vocabulary size).

  Args:
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A scalar representing the vocabulary size.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_get_piece_size(
      model_file=model_file, model_proto=model_proto, name=name)

def piece_to_id(input, model_file=None, model_proto=None, name=None):
  """Converts piece into vocabulary id.

  Args:
    input: An arbitrary tensor of string.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of int32 with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_piece_to_id(
      input, model_file=model_file, model_proto=model_proto, name=name)

def id_to_piece(input, model_file=None, model_proto=None, name=None):
  """Converts vocabulary id into piece.

  Args:
    input: An arbitrary tensor of int32.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of string with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_id_to_piece(
      input, model_file=model_file, model_proto=model_proto, name=name)

def is_unknown(input, model_file=None, model_proto=None, name=None):
  """Returns true if input id is unknown piece.

  Args:
    input: An arbitrary tensor of int32.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of bool with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_get_piece_type(
      input,
      model_file=model_file,
      model_proto=model_proto,
      name=name,
      piece_type=0)

def is_control(input, model_file=None, model_proto=None, name=None):
  """Returns true if input id is control piece.

  Args:
    input: An arbitrary tensor of int32.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of bool with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_get_piece_type(
      input,
      model_file=model_file,
      model_proto=model_proto,
      name=name,
      piece_type=1)

def is_unused(input, model_file=None, model_proto=None, name=None):
  """Returns true if input id is unused piece.

  Args:
    input: An arbitrary tensor of int32.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    name: The name argument that is passed to the op function.

  Returns:
    A tensor of bool with the same shape as input.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_get_piece_type(
      input,
      model_file=model_file,
      model_proto=model_proto,
      name=name,
      piece_type=2)

def encode_dense(input_sentences,
                 nbest_size=0,
                 alpha=1.0,
                 model_file=None,
                 model_proto=None,
                 reverse=False,
                 add_bos=False,
                 add_eos=False,
                 out_type=tf.int32,
                 name=None):
  """Encodes sentences into pieces in dense tensor format.

  Args:
    input_sentences: A 1D string tensor of arbitrary size holding the raw text
      of input sentences.
    nbest_size: A scalar or 1D tensor for sampling.
                nbest_size = {0,1}: No sampling is performed.
                nbest_size > 1: samples from the nbest_size results.
                nbest_size < 0: assuming that nbest_size is infinite and samples
                  from the all hypothesis (lattice) using
                  forward-filtering-and-backward-sampling algorithm.
    alpha: A scalar or 1D tensor for a smoothing parameter. Inverse temperature
      for probability rescaling.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    reverse: Reverses the tokenized sequence (Default = false)
    add_bos: Add <s> to the result (Default = false)
    add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
      reversing (if enabled).
    out_type: output type. tf.int32 or tf.string (Default = tf.int32) Setting
      tf.int32 directly encodes the string into an id sequence.
    name: The name argument that is passed to the op function.

  Returns:
    pieces: A dense 2D tensor representing the tokenized sentences.
    sequence_length: A 1D tensor representing the length of pieces.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_encode_dense(
      input_sentences,
      nbest_size=nbest_size,
      alpha=alpha,
      model_file=model_file,
      model_proto=model_proto,
      reverse=reverse,
      add_bos=add_bos,
      add_eos=add_eos,
      out_type=out_type,
      name=name)

def encode_sparse(input_sentences,
                  nbest_size=0,
                  alpha=1.0,
                  model_file=None,
                  model_proto=None,
                  reverse=False,
                  add_bos=False,
                  add_eos=False,
                  out_type=tf.int32,
                  name=None):
  """Encodes sentences into pieces in sparse tensor format.

  Args:
    input_sentences: A 1D string tensor of arbitrary size holding the raw text
      of input sentences.
    nbest_size: A scalar or 1D tensor for sampling.
                nbest_size = {0,1}: No sampling is performed.
                nbest_size > 1: samples from the nbest_size results.
                nbest_size < 0: assuming that nbest_size is infinite and samples
                  from the all hypothesis (lattice) using
                  forward-filtering-and-backward-sampling algorithm.
    alpha: A scalar or 1D tensor for a smoothing parameter. Inverse temperature
      for probability rescaling.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    reverse: Reverses the tokenized sequence (Default = false)
    add_bos: Add <s> to the result (Default = false)
    add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
      reversing (if enabled).
    out_type: output type. tf.int32 or tf.string (Default = tf.int32) Setting
      tf.int32 directly encodes the string into an id sequence.
    name: The name argument that is passed to the op function.

  Returns:
    pieces: A sparse 2D tensor representing the tokenized sentences.
  """

  indices, values, dense_shape = (
      _gen_sentencepiece_processor_op.sentencepiece_encode_sparse(
          input_sentences,
          nbest_size=nbest_size,
          alpha=alpha,
          model_file=model_file,
          model_proto=model_proto,
          reverse=reverse,
          add_bos=add_bos,
          add_eos=add_eos,
          out_type=out_type,
          name=name))
  return tf.SparseTensor(indices, values, dense_shape)

def decode(pieces,
           sequence_length,
           model_file=None,
           model_proto=None,
           reverse=False,
           name=None):
  """Decode pieces into postprocessed text.

  Args:
    pieces: A 2D int32 or string tensor [batch_size x max_length] of encoded
      sequences.
    sequence_length: A 1D int32 tensor [batch_size] representing the length of
      pieces.
    model_file: The sentencepiece model file path.
    model_proto: The sentencepiece model serialized proto. Either `model_file`
      or `model_proto` must be set.
    reverse: Reverses the tokenized sequence (Default = false)
    name: The name argument that is passed to the op function.

  Returns:
    text: A 1D string tensor of decoded string.
  """

  return _gen_sentencepiece_processor_op.sentencepiece_decode(
      pieces,
      sequence_length,
      model_file=model_file,
      model_proto=model_proto,
      reverse=reverse,
      name=name)

# Adds an alias for encode_dense. Accepts the `encode` function.
encode = encode_dense
sparse_encode = encode_sparse
dense_encode = encode_dense

tf.no_gradient('SentencepieceGetPieceSize')
tf.no_gradient('SentencepieceIdToPiece')
tf.no_gradient('SentencepiecePieceToId')
tf.no_gradient('SentencepieceGetPieceType')
tf.no_gradient('SentencepieceEncodeDense')
tf.no_gradient('SentencepieceEncodeSparse')
tf.no_gradient('SentencepieceDecode')
<|endoftext|>


<commit_before>"""
The evaluating printf(...) after break stop and then up a stack frame.
"""

import os, time
import unittest2
import lldb
from lldbtest import *

class Radar9531204TestCase(TestBase):

    mydir = os.path.join("expression_command", "radar_9531204")

    # rdar://problem/9531204
    @unittest2.expectedFailure
    def test_expr_commands(self):
        """The evaluating printf(...) after break stop and then up a stack frame."""
        self.buildDefault()

        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        self.expect("breakpoint set -n foo",
                    BREAKPOINT_CREATED,
            startstr = "Breakpoint created: 1: name = 'foo', locations = 1")

        self.runCmd("run", RUN_SUCCEEDED)

        self.runCmd("frame variable")

        # This works fine.
        self.runCmd('expression (int)printf("value is: %d.\\n", value);')

        # rdar://problem/9531204
        # "Error dematerializing struct" error when evaluating expressions "up" on the stack
        self.runCmd('up') # frame select -r 1

        self.runCmd("frame variable")

        # This does not currently.
        self.runCmd('expression (int)printf("argc is: %d.\\n", argc)')

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<commit_msg>Remove the expectedFailure decorator.  The test has been passing for some time now.<commit_after>"""
The evaluating printf(...) after break stop and then up a stack frame.
"""

import os, time
import unittest2
import lldb
from lldbtest import *

class Radar9531204TestCase(TestBase):

    mydir = os.path.join("expression_command", "radar_9531204")

    # rdar://problem/9531204
    def test_expr_commands(self):
        """The evaluating printf(...) after break stop and then up a stack frame."""
        self.buildDefault()

        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        self.expect("breakpoint set -n foo",
                    BREAKPOINT_CREATED,
            startstr = "Breakpoint created: 1: name = 'foo', locations = 1")

        self.runCmd("run", RUN_SUCCEEDED)

        self.runCmd("frame variable")

        # This works fine.
        self.runCmd('expression (int)printf("value is: %d.\\n", value);')

        # rdar://problem/9531204
        # "Error dematerializing struct" error when evaluating expressions "up" on the stack
        self.runCmd('up') # frame select -r 1

        self.runCmd("frame variable")

        # This does not currently.
        self.runCmd('expression (int)printf("argc is: %d.\\n", argc)')

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<|endoftext|>


<commit_before>import numpy as np

from foolbox.attacks import GradientSignAttack as Attack

def test_attack(bn_adversarial):
    adv = bn_adversarial
    attack = Attack()
    attack(adv)
    assert adv.perturbed is not None
    assert adv.distance.value < np.inf

def test_attack_eps(bn_adversarial):
    adv = bn_adversarial
    attack = Attack()
    attack(adv, epsilons=np.linspace(0., 1., 100)[1:])
    assert adv.perturbed is not None
    assert adv.distance.value < np.inf

def test_attack_gl(gl_bn_adversarial):
    adv = gl_bn_adversarial
    attack = Attack()
    attack(adv)
    assert adv.perturbed is None
    assert adv.distance.value == np.inf
<commit_msg>updated GradientSignAttack batch attack tests<commit_after>import numpy as np

from foolbox.attacks import GradientSignAttack as Attack

def test_untargeted_attack(bn_model, bn_criterion, bn_images, bn_labels):
    attack = Attack(bn_model, bn_criterion)
    advs = attack(bn_images, bn_labels, unpack=False)
    for adv in advs:
        assert adv.perturbed is not None
        assert adv.distance.value < np.inf

def test_attack_eps(bn_model, bn_criterion, bn_images, bn_labels):
    attack = Attack(bn_model, bn_criterion)
    advs = attack(bn_images, bn_labels, unpack=False, epsilons=np.linspace(0., 1., 100)[1:])
    for adv in advs:
        assert adv.perturbed is not None
        assert adv.distance.value < np.inf

def test_attack_gl(gl_bn_model, bn_criterion, bn_images, bn_labels):
    attack = Attack(gl_bn_model, bn_criterion)
    advs = attack(bn_images, bn_labels, unpack=False)
    for adv in advs:
        assert adv.perturbed is None
        assert adv.distance.value == np.inf
<|endoftext|>