<commit_before>from __future__ import print_function, division, absolute_import
from fontTools.misc.py23 import *
from fontTools.misc.arrayTools import pairwise
from fontTools.pens.filterPen import ContourFilterPen

__all__ = ["reversedContour", "ReverseContourPen"]

class ReverseContourPen(ContourFilterPen):
    """Filter pen that passes outline data to another pen, but reversing
    the winding direction of all contours. Components are simply passed
    through unchanged.

    Closed contours are reversed in such a way that the first point remains
    the first point.
    """

    def filterContour(self, contour):
        return reversedContour(contour)

def reversedContour(contour):
    """ Generator that takes a list of pen's (operator, operands) tuples,
    and yields them with the winding direction reversed.
    """
    if not contour:
        return  # nothing to do, stop iteration

    # valid contours must have at least a starting and ending command,
    # can't have one without the other
    assert len(contour) > 1, "invalid contour"

    # the type of the last command determines if the contour is closed
    contourType = contour.pop()[0]
    assert contourType in ("endPath", "closePath")
    closed = contourType == "closePath"

    firstType, firstPts = contour.pop(0)
    assert firstType in ("moveTo", "qCurveTo"), (
        "invalid initial segment type: %r" % firstType)
    firstOnCurve = firstPts[-1]
    if firstType == "qCurveTo":
        # special case for TrueType paths contaning only off-curve points
        assert firstOnCurve is None, (
            "off-curve only paths must end with 'None'")
        assert not contour, (
            "only one qCurveTo allowed per off-curve path")
        firstPts = ((firstPts[0],) + tuple(reversed(firstPts[1:-1])) +
                    (None,))

    if not contour:
        # contour contains only one segment, nothing to reverse
        if firstType == "moveTo":
            closed = False  # single-point paths can't be closed
        else:
            closed = True  # off-curve paths are closed by definition
        yield firstType, firstPts
    else:
        lastType, lastPts = contour[-1]
        lastOnCurve = lastPts[-1]
        if closed:
            # for closed paths, we keep the starting point
            yield firstType, firstPts
            if firstOnCurve != lastOnCurve:
                # emit an implied line between the last and first points
                yield "lineTo", (lastOnCurve,)
                contour[-1] = (lastType,
                               tuple(lastPts[:-1]) + (firstOnCurve,))

            # if a lineTo follows the initial moveTo, after reversing it
            # will be implied by the closePath, so we don't emit one
            secondType, secondPts = contour[0]
            if secondType == "lineTo":
                del contour[0]
                if contour:
                    contour[-1] = (lastType,
                                   tuple(lastPts[:-1]) + secondPts)
        else:
            # for open paths, the last point will become the first
            yield firstType, (lastOnCurve,)
            contour[-1] = (lastType, tuple(lastPts[:-1]) + (firstOnCurve,))

        # we iterate over all segment pairs in reverse order, and yield
        # each one with the off-curve points reversed (if any), and
        # with the on-curve point of the following segment
        for (curType, curPts), (_, nextPts) in pairwise(
                contour, reverse=True):
            yield curType, tuple(reversed(curPts[:-1])) + (nextPts[-1],)

    yield "closePath" if closed else "endPath", ()
<commit_msg>[reverseContourPen] don't imply closing lineTo when same as moveTo<commit_after>from __future__ import print_function, division, absolute_import
from fontTools.misc.py23 import *
from fontTools.misc.arrayTools import pairwise
from fontTools.pens.filterPen import ContourFilterPen

__all__ = ["reversedContour", "ReverseContourPen"]

class ReverseContourPen(ContourFilterPen):
    """Filter pen that passes outline data to another pen, but reversing
    the winding direction of all contours. Components are simply passed
    through unchanged.

    Closed contours are reversed in such a way that the first point remains
    the first point.
    """

    def filterContour(self, contour):
        return reversedContour(contour)

def reversedContour(contour):
    """ Generator that takes a list of pen's (operator, operands) tuples,
    and yields them with the winding direction reversed.
    """
    if not contour:
        return  # nothing to do, stop iteration

    # valid contours must have at least a starting and ending command,
    # can't have one without the other
    assert len(contour) > 1, "invalid contour"

    # the type of the last command determines if the contour is closed
    contourType = contour.pop()[0]
    assert contourType in ("endPath", "closePath")
    closed = contourType == "closePath"

    firstType, firstPts = contour.pop(0)
    assert firstType in ("moveTo", "qCurveTo"), (
        "invalid initial segment type: %r" % firstType)
    firstOnCurve = firstPts[-1]
    if firstType == "qCurveTo":
        # special case for TrueType paths contaning only off-curve points
        assert firstOnCurve is None, (
            "off-curve only paths must end with 'None'")
        assert not contour, (
            "only one qCurveTo allowed per off-curve path")
        firstPts = ((firstPts[0],) + tuple(reversed(firstPts[1:-1])) +
                    (None,))

    if not contour:
        # contour contains only one segment, nothing to reverse
        if firstType == "moveTo":
            closed = False  # single-point paths can't be closed
        else:
            closed = True  # off-curve paths are closed by definition
        yield firstType, firstPts
    else:
        lastType, lastPts = contour[-1]
        lastOnCurve = lastPts[-1]
        if closed:
            # for closed paths, we keep the starting point
            yield firstType, firstPts
            if firstOnCurve != lastOnCurve:
                # emit an implied line between the last and first points
                yield "lineTo", (lastOnCurve,)
                contour[-1] = (lastType,
                               tuple(lastPts[:-1]) + (firstOnCurve,))

            if len(contour) > 1:
                secondType, secondPts = contour[0]
            else:
                # contour has only two points, the second and last are the same
                secondType, secondPts = lastType, lastPts
            # if a lineTo follows the initial moveTo, after reversing it
            # will be implied by the closePath, so we don't emit one;
            # unless the lineTo and moveTo overlap, in which case we keep the
            # duplicate points
            if secondType == "lineTo" and firstPts != secondPts:
                del contour[0]
                if contour:
                    contour[-1] = (lastType,
                                   tuple(lastPts[:-1]) + secondPts)
        else:
            # for open paths, the last point will become the first
            yield firstType, (lastOnCurve,)
            contour[-1] = (lastType, tuple(lastPts[:-1]) + (firstOnCurve,))

        # we iterate over all segment pairs in reverse order, and yield
        # each one with the off-curve points reversed (if any), and
        # with the on-curve point of the following segment
        for (curType, curPts), (_, nextPts) in pairwise(
                contour, reverse=True):
            yield curType, tuple(reversed(curPts[:-1])) + (nextPts[-1],)

    yield "closePath" if closed else "endPath", ()
<|endoftext|>


<commit_before>#===============================================================================
# Copyright 2013 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#===============================================================================

#============= enthought library imports =======================
import time
from chaco.array_data_source import ArrayDataSource
from chaco.scales.time_scale import CalendarScaleSystem
from chaco.scales_tick_generator import ScalesTickGenerator
from traits.api import Array
#============= standard library imports ========================
from numpy import array, Inf
#============= local library imports  ==========================
from pychron.processing.plotters.arar_figure import BaseArArFigure

N = 500

class Series(BaseArArFigure):
    xs = Array
    _omit_key = 'omit_series'

    def max_x(self, *args):
        if len(self.xs):
            return max(self.xs)
        return -Inf

    def min_x(self, *args):
        if len(self.xs):
            return min(self.xs)
        return Inf

    def mean_x(self, *args):
        if len(self.xs):
            return self.xs.mean()
        return 0

    def build(self, plots):
        graph = self.graph
        for po in plots:
            if po.use:
                p = graph.new_plot(padding=self.padding,
                                   ytitle=po.name,
                                   xtitle='Time')

                p.value_scale = po.scale
                p.padding_left = 75
                p.value_range.tight_bounds = False

    def plot(self, plots):
        """
            plot data on plots
        """

        omits=self._get_omitted(self.sorted_analyses, omit='omit_series')
        # print omits
        # omits=[0,2]
        graph = self.graph

        xs = array([ai.timestamp for ai in self.sorted_analyses])
        #print plots, xs
        if plots:
            px = plots[0]

            if px.normalize == 'now':
                norm = time.time()
            else:
                norm = xs[-1]

            xs -= norm
            if not px.use_time_axis:
                xs /= 3600
            else:
                graph.convert_index_func = lambda x: '{:0.2f} hrs'.format(x / 3600.)

            self.xs = xs

            with graph.no_regression(refresh=True):
                plots = [po for po in plots if po.use]
                for i, po in enumerate(plots):
                    self._plot_series(po, i, omits)

                if plots:
                    graph.set_x_limits(min_=min(self.xs), max_=max(self.xs), pad='0.1',
                                       plotid=0)

    def _plot_series(self, po, pid, omits):
        graph = self.graph
        try:
            ys = [ai.nominal_value for ai in self._unpack_attr(po.name)]
            yerr = [ai.std_dev for ai in self._unpack_attr(po.name)]

            n = [ai.record_id for ai in self.sorted_analyses]

            args = graph.new_series(x=self.xs,
                                    y=ys,
                                    display_index=ArrayDataSource(data=n),
                                    yerror=yerr,
                                    fit=po.fit,
                                    plotid=pid,
                                    type='scatter')
            if len(args) == 2:
                scatter, p = args
            else:
                p, scatter, l = args

            # sel=scatter.index.metadata.get('selections',[])
            # sel+=omits
            # scatter.index.metadata['selections']=list(set(sel))

            if po.use_time_axis:
                p.x_axis.tick_generator = ScalesTickGenerator(scale=CalendarScaleSystem())

            #p.value_scale = po.scale
            self._add_error_bars(scatter, yerr, 'y', 1,
                                 visible=po.y_error)
        except (KeyError, ZeroDivisionError), e:
            print 'Series', e

    def _unpack_attr(self, attr):
        if attr.endswith('bs'):
            # f=lambda x: x.baseline.uvalue
            return (ai.get_baseline(attr).uvalue for ai in self.sorted_analyses)

        elif attr == 'PC':
            return super(Series, self)._unpack_attr(attr)
        else:
            gs = super(Series, self)._unpack_attr(attr)
            f = lambda x: x.get_intensity()
            return map(f, gs)

            #if attr.endswith('bs'):
            #    return (ai.isotopes[attr[:-2]].baseline.uvalue
            #            for ai in self.sorted_analyses)
            ##elif '/' in attr:
            ##    n, d = attr.split('/')
            #    #return (getattr(ai, n) / getattr(ai, d)
            #    #        for ai in self.sorted_analyses)
            #elif attr == 'PC':
            #    return (getattr(ai, 'peak_center')
            #            for ai in self.sorted_analyses)
            #else:
            #    return super(Series, self)._unpack_attr(attr)

#===============================================================================
# plotters
#===============================================================================

#===============================================================================
# overlays
#===============================================================================

#===============================================================================
# utils
#===============================================================================

#===============================================================================
# labels
#===============================================================================

#============= EOF =============================================
<commit_msg>enabled omit series<commit_after>#===============================================================================
# Copyright 2013 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#===============================================================================

#============= enthought library imports =======================
import time
from chaco.array_data_source import ArrayDataSource
from chaco.scales.time_scale import CalendarScaleSystem
from chaco.scales_tick_generator import ScalesTickGenerator
from traits.api import Array
#============= standard library imports ========================
from numpy import array, Inf
#============= local library imports  ==========================
from pychron.processing.plotters.arar_figure import BaseArArFigure

N = 500

class Series(BaseArArFigure):
    xs = Array
    _omit_key = 'omit_series'

    def max_x(self, *args):
        if len(self.xs):
            return max(self.xs)
        return -Inf

    def min_x(self, *args):
        if len(self.xs):
            return min(self.xs)
        return Inf

    def mean_x(self, *args):
        if len(self.xs):
            return self.xs.mean()
        return 0

    def build(self, plots):
        graph = self.graph
        for po in plots:
            if po.use:
                p = graph.new_plot(padding=self.padding,
                                   ytitle=po.name,
                                   xtitle='Time')

                p.value_scale = po.scale
                p.padding_left = 75
                p.value_range.tight_bounds = False

    def plot(self, plots):
        """
            plot data on plots
        """

        omits=self._get_omitted(self.sorted_analyses, omit='omit_series')
        graph = self.graph

        xs = array([ai.timestamp for ai in self.sorted_analyses])
        #print plots, xs
        if plots:
            px = plots[0]

            if px.normalize == 'now':
                norm = time.time()
            else:
                norm = xs[-1]

            xs -= norm
            if not px.use_time_axis:
                xs /= 3600
            else:
                graph.convert_index_func = lambda x: '{:0.2f} hrs'.format(x / 3600.)

            self.xs = xs

            with graph.no_regression(refresh=True):
                plots = [po for po in plots if po.use]
                for i, po in enumerate(plots):
                    self._plot_series(po, i, omits)

                if plots:
                    graph.set_x_limits(min_=min(self.xs), max_=max(self.xs), pad='0.1',
                                       plotid=0)

    def _plot_series(self, po, pid, omits):
        graph = self.graph
        try:
            ys = [ai.nominal_value for ai in self._unpack_attr(po.name)]
            yerr = [ai.std_dev for ai in self._unpack_attr(po.name)]

            n = [ai.record_id for ai in self.sorted_analyses]

            args = graph.new_series(x=self.xs,
                                    y=ys,
                                    display_index=ArrayDataSource(data=n),
                                    yerror=yerr,
                                    fit=po.fit,
                                    plotid=pid,
                                    type='scatter')
            if len(args) == 2:
                scatter, p = args
            else:
                p, scatter, l = args

            sel=scatter.index.metadata.get('selections',[])
            sel+=omits
            scatter.index.metadata['selections']=list(set(sel))

            if po.use_time_axis:
                p.x_axis.tick_generator = ScalesTickGenerator(scale=CalendarScaleSystem())

            #p.value_scale = po.scale
            self._add_error_bars(scatter, yerr, 'y', 1,
                                 visible=po.y_error)
        except (KeyError, ZeroDivisionError), e:
            print 'Series', e

    def _unpack_attr(self, attr):
        if attr.endswith('bs'):
            # f=lambda x: x.baseline.uvalue
            return (ai.get_baseline(attr).uvalue for ai in self.sorted_analyses)

        elif attr == 'PC':
            return super(Series, self)._unpack_attr(attr)
        else:
            gs = super(Series, self)._unpack_attr(attr)
            f = lambda x: x.get_intensity()
            return map(f, gs)

            #if attr.endswith('bs'):
            #    return (ai.isotopes[attr[:-2]].baseline.uvalue
            #            for ai in self.sorted_analyses)
            ##elif '/' in attr:
            ##    n, d = attr.split('/')
            #    #return (getattr(ai, n) / getattr(ai, d)
            #    #        for ai in self.sorted_analyses)
            #elif attr == 'PC':
            #    return (getattr(ai, 'peak_center')
            #            for ai in self.sorted_analyses)
            #else:
            #    return super(Series, self)._unpack_attr(attr)

#===============================================================================
# plotters
#===============================================================================

#===============================================================================
# overlays
#===============================================================================

#===============================================================================
# utils
#===============================================================================

#===============================================================================
# labels
#===============================================================================

#============= EOF =============================================
<|endoftext|>


<commit_before># -*- coding: iso-8859-15 -*-
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2012 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================

from django.db import models
from django.db import connection
from django.db.models import Avg, Max, Min, Count
from django.conf import settings

from pycsw import util
from geonode.base.models import ResourceBase

class GeoNodeRepository(object):
    ''' Class to interact with underlying repository '''
    def __init__(self, context, repo_filter=None):
        ''' Initialize repository '''

        self.context = context
        self.filter = repo_filter

        self.dbtype = settings.DATABASES['default']['ENGINE'].split('.')[-1]

        # GeoNode PostgreSQL installs are PostGIS enabled
        if self.dbtype == 'postgresql_psycopg2':
            self.dbtype = 'postgresql+postgis'

        if self.dbtype in ['sqlite', 'sqlite3']:  # load SQLite query bindings
            cursor = connection.cursor()
            connection.connection.create_function(
            'query_spatial', 4, util.query_spatial)
            connection.connection.create_function(
            'get_anytext', 1, util.get_anytext)
            connection.connection.create_function(
            'get_geometry_area', 1, util.get_geometry_area)

        # generate core queryables db and obj bindings
        self.queryables = {}

        for tname in self.context.model['typenames']:
            for qname in self.context.model['typenames'][tname]['queryables']:
                self.queryables[qname] = {}

                for qkey, qvalue in \
                self.context.model['typenames'][tname]['queryables'][qname].iteritems():
                    self.queryables[qname][qkey] = qvalue

        # flatten all queryables
        # TODO smarter way of doing this
        self.queryables['_all'] = {}
        for qbl in self.queryables:
            self.queryables['_all'].update(self.queryables[qbl])
        self.queryables['_all'].update(self.context.md_core_model['mappings'])

    def query_ids(self, ids):
        ''' Query by list of identifiers '''
        return self._get_repo_filter(ResourceBase.objects).filter(uuid__in=ids).all()

    def query_domain(self, domain, typenames, domainquerytype='list',
        count=False):
        ''' Query by property domain values '''

        objects = self._get_repo_filter(ResourceBase.objects)

        if domainquerytype == 'range':
            return [tuple(objects.aggregate(
            Min(domain), Max(domain)).values())]
        else:
            if count:
                return [(d[domain], d['%s__count' % domain]) \
                for d in objects.values(domain).annotate(Count(domain))]
            else:
                return objects.values_list(domain).distinct()

    def query_latest_insert(self):
        ''' Query to get latest update to repository '''
        from datetime import datetime
        return self._get_repo_filter(ResourceBase.objects).aggregate(
            Max('date'))['date__max'].strftime('%Y-%m-%dT%H:%M:%SZ')

    def query_source(self, source):
        ''' Query by source '''
        return self._get_repo_filter(ResourceBase.objects).filter(source=source)

    def query(self, constraint, sortby=None, typenames=None,
        maxrecords=10, startposition=0):
        ''' Query records from underlying repository '''

        # run the raw query and get total
        if 'where' in constraint:  # GetRecords with constraint
            query = self._get_repo_filter(ResourceBase.objects).extra(where=[constraint['where']], params=constraint['values'])

        else:  # GetRecords sans constraint
            query = self._get_repo_filter(ResourceBase.objects)

        total = query.count()

        # apply sorting, limit and offset
        if sortby is not None:
            if 'spatial' in sortby and sortby['spatial']:  # spatial sort
                desc = False
                if sortby['order'] == 'DESC':
                    desc = True
                query = query.all()
                return [str(total), sorted(query, key=lambda x: float(util.get_geometry_area(getattr(x, sortby['propertyname']))), reverse=desc)[startposition:maxrecords]]
            else:
                if sortby['order'] == 'DESC':
                    pname = '-%s' % sortby['propertyname']
                else:
                    pname = sortby['propertyname']
                return [str(total), \
                query.order_by(pname)[startposition:maxrecords]]
        else:  # no sort
            return [str(total), query.all()[startposition:maxrecords]]

    def _get_repo_filter(self, query):
        ''' Apply repository wide side filter / mask query '''
        if self.filter is not None:
            return query.filter(self.filter)
        return query
<commit_msg>Update geonode_.py<commit_after># -*- coding: iso-8859-15 -*-
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2012 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================

from django.db import models
from django.db import connection
from django.db.models import Avg, Max, Min, Count
from django.conf import settings

from pycsw import util
from geonode.base.models import ResourceBase

class GeoNodeRepository(object):
    ''' Class to interact with underlying repository '''
    def __init__(self, context, repo_filter=None):
        ''' Initialize repository '''

        self.context = context
        self.filter = repo_filter

        self.dbtype = settings.DATABASES['default']['ENGINE'].split('.')[-1]

        # GeoNode PostgreSQL installs are PostGIS enabled
        if self.dbtype == 'postgresql_psycopg2':
            self.dbtype = 'postgresql+postgis+wkt'

        if self.dbtype in ['sqlite', 'sqlite3']:  # load SQLite query bindings
            cursor = connection.cursor()
            connection.connection.create_function(
            'query_spatial', 4, util.query_spatial)
            connection.connection.create_function(
            'get_anytext', 1, util.get_anytext)
            connection.connection.create_function(
            'get_geometry_area', 1, util.get_geometry_area)

        # generate core queryables db and obj bindings
        self.queryables = {}

        for tname in self.context.model['typenames']:
            for qname in self.context.model['typenames'][tname]['queryables']:
                self.queryables[qname] = {}

                for qkey, qvalue in \
                self.context.model['typenames'][tname]['queryables'][qname].iteritems():
                    self.queryables[qname][qkey] = qvalue

        # flatten all queryables
        # TODO smarter way of doing this
        self.queryables['_all'] = {}
        for qbl in self.queryables:
            self.queryables['_all'].update(self.queryables[qbl])
        self.queryables['_all'].update(self.context.md_core_model['mappings'])

    def query_ids(self, ids):
        ''' Query by list of identifiers '''
        return self._get_repo_filter(ResourceBase.objects).filter(uuid__in=ids).all()

    def query_domain(self, domain, typenames, domainquerytype='list',
        count=False):
        ''' Query by property domain values '''

        objects = self._get_repo_filter(ResourceBase.objects)

        if domainquerytype == 'range':
            return [tuple(objects.aggregate(
            Min(domain), Max(domain)).values())]
        else:
            if count:
                return [(d[domain], d['%s__count' % domain]) \
                for d in objects.values(domain).annotate(Count(domain))]
            else:
                return objects.values_list(domain).distinct()

    def query_latest_insert(self):
        ''' Query to get latest update to repository '''
        from datetime import datetime
        return self._get_repo_filter(ResourceBase.objects).aggregate(
            Max('date'))['date__max'].strftime('%Y-%m-%dT%H:%M:%SZ')

    def query_source(self, source):
        ''' Query by source '''
        return self._get_repo_filter(ResourceBase.objects).filter(source=source)

    def query(self, constraint, sortby=None, typenames=None,
        maxrecords=10, startposition=0):
        ''' Query records from underlying repository '''

        # run the raw query and get total
        if 'where' in constraint:  # GetRecords with constraint
            query = self._get_repo_filter(ResourceBase.objects).extra(where=[constraint['where']], params=constraint['values'])

        else:  # GetRecords sans constraint
            query = self._get_repo_filter(ResourceBase.objects)

        total = query.count()

        # apply sorting, limit and offset
        if sortby is not None:
            if 'spatial' in sortby and sortby['spatial']:  # spatial sort
                desc = False
                if sortby['order'] == 'DESC':
                    desc = True
                query = query.all()
                return [str(total), sorted(query, key=lambda x: float(util.get_geometry_area(getattr(x, sortby['propertyname']))), reverse=desc)[startposition:maxrecords]]
            else:
                if sortby['order'] == 'DESC':
                    pname = '-%s' % sortby['propertyname']
                else:
                    pname = sortby['propertyname']
                return [str(total), \
                query.order_by(pname)[startposition:maxrecords]]
        else:  # no sort
            return [str(total), query.all()[startposition:maxrecords]]

    def _get_repo_filter(self, query):
        ''' Apply repository wide side filter / mask query '''
        if self.filter is not None:
            return query.filter(self.filter)
        return query
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
This module tests some methods related to ``CDS`` format
reader/writer.
Requires `pyyaml <https://pyyaml.org/>`_ to be installed.
"""
from io import StringIO

from astropy.table import Table
from astropy.io import ascii
from astropy import units as u

def test_write_data():
    output = ['S08-229 4625 1.23 1.23 -1.50      ',
              'S05-10  4342 0.91 1.82 -2.11  0.14',
              'S05-47  4654 1.28 1.74 -1.64  0.16']

    dat = 'data/cdsFunctional2.dat'
    t = Table.read(dat, format='ascii.cds')

    out = StringIO()
    t.write(out, format='ascii.cds')
    # get the last section of table which will be the data.
    lines = out.getvalue().splitlines()
    i_secs = [i for i, s in enumerate(lines)
             if s.startswith(('------', '======='))]
    lines = lines[i_secs[-1]+1:]
    assert lines == output

test_dat = ['names e d s i',
            'HD81809 1E-7 22.25608 +2 67',
            'HD103095 -31.6e5 +27.2500 -9E34 -30']

def test_write_ByteByByte_units():
    t = ascii.read(test_dat)
    colUnits = [None, u.C, u.kg, None, u.year]
    t._set_column_attribute('unit', colUnits)
    # add a column with magnitude units.
    # note that magnitude has to be assigned for each value explicitly.
    t['magnitude'] = [u.Magnitude(25), u.Magnitude(-9)]
    colUnits.append(u.mag)
    out = StringIO()
    t.write(out, format='ascii.cds')
    # read written table.
    tRead = ascii.read(out.getvalue(), format='cds')
    assert [tRead[col].unit for col in tRead.columns] == colUnits
<commit_msg>add more tests for CDS default header and for ByteByByte description of columns with Null values [skip ci]<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
This module tests some methods related to ``CDS`` format
reader/writer.
Requires `pyyaml <https://pyyaml.org/>`_ to be installed.
"""
from io import StringIO

from astropy.table import Table
from astropy.io import ascii
from astropy import units as u

test_dat = ['names e d s i',
            'HD81809 1E-7 22.25608 +2 67',
            'HD103095 -31.6e5 +27.2500 -9E34 -30']

def test_write_data():
    expOutput = ['S08-229 4625 1.23 1.23 -1.50      ',
                 'S05-10  4342 0.91 1.82 -2.11  0.14',
                 'S05-47  4654 1.28 1.74 -1.64  0.16']

    dat = 'data/cdsFunctional2.dat'
    t = Table.read(dat, format='ascii.cds')

    out = StringIO()
    t.write(out, format='ascii.cds')
    # get the last section of table which will be the data.
    lines = out.getvalue().splitlines()
    i_secs = [i for i, s in enumerate(lines)
              if s.startswith(('------', '======='))]
    lines = lines[i_secs[-1]+1:]
    assert lines == expOutput

def test_write_ByteByByte_units():
    t = ascii.read(test_dat)
    colUnits = [None, u.C, u.kg, None, u.year]
    t._set_column_attribute('unit', colUnits)
    # add a column with magnitude units.
    # note that magnitude has to be assigned for each value explicitly.
    t['magnitude'] = [u.Magnitude(25), u.Magnitude(-9)]
    colUnits.append(u.mag)
    out = StringIO()
    t.write(out, format='ascii.cds')
    # read written table.
    tRead = ascii.read(out.getvalue(), format='cds')
    assert [tRead[col].unit for col in tRead.columns] == colUnits

def test_write_readme_with_default_options():
    expOutput = '''\
                                                      (1st author ?, Date ?)
================================================================================
Title ?
    Authors ?
    ref ?
================================================================================
Keywords:

Objects:
    -----------------------------------------
       RA   (2000)   DE    Designation(s)
    -----------------------------------------

Abstract:
  Abstract ?

File Summary:
--------------------------------------------------------------------------------
 FileName    Lrecl   Records    Explanations
--------------------------------------------------------------------------------
ReadMe         80        . this file
table          35        2

--------------------------------------------------------------------------------
Byte-by-byte Description of file: table.dat
--------------------------------------------------------------------------------
 Bytes Format Units  Label     Explanations
--------------------------------------------------------------------------------
 1- 8   A8     ---    names    Description of names
10-14   E5.1   ---    e       [-3160000.0/0.01] Description of e
16-23   F8.5   ---    d       [22.25/27.25] Description of d
25-31   E7.1   ---    s       [-9e+34/2.0] Description of s
33-35   I3     ---    i       [-30/67] Description of i

--------------------------------------------------------------------------------

See also:

Acknowledgements:

References:
================================================================================
     (prepared by author  / pyreadme )
--------------------------------------------------------------------------------
HD81809  1e-07  22.25608   2e+00  67
HD103095 -3e+06 27.25000  -9e+34 -30
'''
    t = ascii.read(test_dat)
    out = StringIO()
    t.write(out, format='ascii.cds')
    assert out.getvalue() == expOutput

def test_write_empty_table():
    out = StringIO()
    import pytest
    with pytest.raises(ValueError):
        Table().write(out, format='ascii.cds')

def test_write_null_data_values():
    expOutput = ['HD81809  1e-07   22.25608  2.0e+00  67',
                 'HD103095 -3e+06  27.25000 -9.0e+34 -30',
                 'Sun                        5.3e+27    ']
    t = ascii.read(test_dat)
    t.add_row(['Sun', '3.25', '0', '5.3e27', '2'],
                mask=[False, True, True, False, True])
    out = StringIO()
    t.write(out, format='ascii.cds')
    lines = out.getvalue().splitlines()
    i_secs = [i for i, s in enumerate(lines)
              if s.startswith(('------', '======='))]
    lines = lines[i_secs[-1]+1:]
    assert lines == expOutput

def test_write_ByteByByte_for_masked_column():
    """
    This test differs from the ``test_write_null_data_values``
    above in that it tests the column value limits in the ByteByByte
    description section for columns whose values are masked.
    It also checks the description for columns with same values.
    """
    expOutput = '''\
--------------------------------------------------------------------------------
Byte-by-byte Description of file: table.dat
--------------------------------------------------------------------------------
 Bytes Format Units  Label     Explanations
--------------------------------------------------------------------------------
 1- 8   A8     ---    names    Description of names
10-14   E5.1   ---    e       [0.0/0.01]? Description of e
16-18   F3.0   ---    d       ? Description of d
20-26   E7.1   ---    s       [-9e+34/2.0] Description of s
28-30   I3     ---    i       [-30/67] Description of i
32-34   F3.1   ---    sameF   [5.0/5.0] Description of sameF
36-37   I2     ---    sameI   [20] Description of sameI

--------------------------------------------------------------------------------

See also:

Acknowledgements:

References:
================================================================================
     (prepared by author  / pyreadme )
--------------------------------------------------------------------------------
HD81809  1e-07    2e+00  67 5.0 20
HD103095         -9e+34 -30 5.0 20
'''
    from astropy.table import MaskedColumn
    t = ascii.read(test_dat)
    t.add_column([5.0, 5.0], name='sameF')
    t.add_column([20, 20], name='sameI')
    t['e'] = MaskedColumn(t['e'], mask=[False, True])
    t['d'] = MaskedColumn(t['d'], mask=[True, True])
    out = StringIO()
    t.write(out, format='ascii.cds')
    lines = out.getvalue().splitlines()
    i_secs = [i for i, s in enumerate(lines)
              if s.startswith(('------', '======='))]
    lines = lines[i_secs[-6]:]
    assert lines == expOutput.splitlines()
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from contextlib import contextmanager
import os
import socket
import numpy as np
from numpy.testing import assert_allclose

from astropy.extern import six
from astropy.io import fits
from astropy.table import Column, Table
from astropy.tests.helper import pytest

from ... import sdss
from ...utils.testing_tools import MockResponse
from ...exceptions import TimeoutError
from ...utils import commons

# actual spectra/data are a bit heavy to include in astroquery, so we don't try
# to deal with them.  Would be nice to find a few very small examples

DATA_FILES = {'spectra_id': 'xid_sp.txt',
              'images_id': 'xid_im.txt',
              'spectra': 'emptyfile.fits',
              'images': 'emptyfile.fits'}

@pytest.fixture
def patch_get(request):
    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(sdss.SDSS, '_request', get_mockreturn)
    return mp

@pytest.fixture
def patch_post(request):
    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(sdss.SDSS, '_request', post_mockreturn)
    return mp

@pytest.fixture
def patch_get_slow(request):
    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(sdss.SDSS, '_request', get_mockreturn_slow)
    return mp

@pytest.fixture
def patch_get_readable_fileobj(request):
    @contextmanager
    def get_readable_fileobj_mockreturn(filename, **kwargs):
        file_obj = data_path(DATA_FILES['spectra'])  # TODO: add images option
        encoding = kwargs.get('encoding', None)
        if encoding == 'binary':
            yield open(file_obj, 'rb')
        else:
            yield open(file_obj, 'r', encoding=encoding)

    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(commons, 'get_readable_fileobj',
               get_readable_fileobj_mockreturn)
    return mp

@pytest.fixture
def patch_get_readable_fileobj_slow(request):
    @contextmanager
    def get_readable_fileobj_mockreturn(filename, **kwargs):
        e = six.moves.urllib_error.URLError('timeout')
        e.reason = socket.timeout()
        raise e
        yield True
    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(commons, 'get_readable_fileobj',
               get_readable_fileobj_mockreturn)
    return mp

def get_mockreturn(method, url, params=None, timeout=10, cache=True, **kwargs):
    if 'SpecObjAll' in params['cmd']:
        filename = data_path(DATA_FILES['spectra_id'])
    else:
        filename = data_path(DATA_FILES['images_id'])
    content = open(filename, 'rb').read()
    return MockResponse(content, **kwargs)

def get_mockreturn_slow(method, url, params=None, timeout=0, **kwargs):
    raise TimeoutError

def post_mockreturn(method, url, params=None, timeout=0, **kwargs):
    filename = data_path(DATA_FILES['images_id'])
    content = open(filename, 'rb').read()
    return MockResponse(content)

def data_path(filename):
    data_dir = os.path.join(os.path.dirname(__file__), 'data')
    return os.path.join(data_dir, filename)

# Test Case: A Seyfert 1 galaxy
coords = commons.ICRSCoordGenerator('0h8m05.63s +14d50m23.3s')

# Test Case: list of coordinates
coords_list = [coords, coords]

# Test Case: Column of coordinates
coords_column = Column(coords_list, name='coordinates')

# We are not testing queries for DR11 because it is not easily available to
# query: "DR11 data are distributed primarily to provide reproducibility of
# published results based on the DR11 data set. As such, not all data-access
# interfaces are supported for DR11."
def url_tester(data_release):
    if data_release < 10:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/search/x_sql.asp'
    if data_release == 10:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/search/x_sql.aspx'
    if data_release >= 12:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/search/x_results.aspx'
    assert sdss.SDSS._last_url == baseurl.format(data_release)

def url_tester_crossid(data_release):
    if data_release < 11:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/crossid/x_crossid.aspx'
    if data_release >= 12:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/search/X_Results.aspx'
    assert sdss.SDSS._last_url == baseurl.format(data_release)

def compare_xid_data(xid, data):
    if six.PY3:
        pytest.xfail('xid/data comparison fails in PY3 because the instrument '
                     'column is bytes in xid and str in data')
    else:
        for col in xid.colnames:
            if xid[col].dtype.type is np.string_:
                assert xid[col] == data[col]
            else:
                assert_allclose(xid[col], data[col])

def image_tester(images, filetype):
    """Test that an image/spectrum is our fake data."""
    assert type(images) == list
    data = fits.open(data_path(DATA_FILES[filetype]))
    assert images[0][0].header == data[0].header
    assert images[0][0].data == data[0].data

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_spectrum(patch_get, patch_get_readable_fileobj, dr,
                       coords=coords):
    xid = sdss.SDSS.query_region(coords, data_release=dr, spectro=True)
    url_tester(dr)
    sp = sdss.SDSS.get_spectra(matches=xid, data_release=dr)
    image_tester(sp, 'spectra')
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_spectrum_mjd(patch_get, patch_get_readable_fileobj, dr):
    sp = sdss.SDSS.get_spectra(plate=2345, fiberID=572, data_release=dr)
    image_tester(sp, 'spectra')

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_spectrum_coords(patch_get, patch_get_readable_fileobj, dr,
                              coords=coords):
    sp = sdss.SDSS.get_spectra(coords, data_release=dr)
    image_tester(sp, 'spectra')

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_sql(patch_get, patch_get_readable_fileobj, dr):
    query = """
            select top 10
              z, ra, dec, bestObjID
            from
              specObj
            where
              class = 'galaxy'
              and z > 0.3
              and zWarning = 0
            """
    xid = sdss.SDSS.query_sql(query, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_image_from_query_region(patch_get, patch_get_readable_fileobj,
                                      dr, coords=coords):
    xid = sdss.SDSS.query_region(coords, data_release=dr)
    # TODO test what img is
    img = sdss.SDSS.get_images(matches=xid)
    image_tester(img, 'images')
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_image_run(patch_get, patch_get_readable_fileobj, dr):
    img = sdss.SDSS.get_images(run=1904, camcol=3, field=164, data_release=dr)
    image_tester(img, 'images')

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_image_coord(patch_get, patch_get_readable_fileobj, dr,
                          coord=coords):
    img = sdss.SDSS.get_images(coords, data_release=dr)
    image_tester(img, 'images')

def test_sdss_template(patch_get, patch_get_readable_fileobj):
    template = sdss.SDSS.get_spectral_template('qso')
    image_tester(template, 'spectra')

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_specobj(patch_get, dr):
    xid = sdss.SDSS.query_specobj(plate=2340, data_release=dr)
    data = Table.read(data_path(DATA_FILES['spectra_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_photoobj(patch_get, dr):
    xid = sdss.SDSS.query_photoobj(
        run=1904, camcol=3, field=164, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_list_coordinates(patch_get, dr):
    xid = sdss.SDSS.query_region(coords_list, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_column_coordinates(patch_get, dr):
    xid = sdss.SDSS.query_region(coords_column, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)
    url_tester(dr)

def test_query_timeout(patch_get_slow, coord=coords):
    with pytest.raises(TimeoutError):
        sdss.SDSS.query_region(coords, timeout=1)

def test_spectra_timeout(patch_get, patch_get_readable_fileobj_slow):
    with pytest.raises(TimeoutError):
        sdss.SDSS.get_spectra(plate=2345, fiberID=572)

def test_images_timeout(patch_get, patch_get_readable_fileobj_slow):
    with pytest.raises(TimeoutError):
        sdss.SDSS.get_images(run=1904, camcol=3, field=164)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_query_crossid(patch_post, dr):
    xid = sdss.SDSS.query_crossid(coords_column, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    assert all(xid == data)
    url_tester_crossid(dr)

# ===========
# Payload tests

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_list_coordinates_payload(dr):
    expect = ("SELECT DISTINCT "
              "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field "
              "FROM PhotoObjAll AS p   WHERE "
              "((p.ra between 2.02291 and 2.02402) and "
              "(p.dec between 14.8393 and 14.8404)) or "
              "((p.ra between 2.02291 and 2.02402) and "
              "(p.dec between 14.8393 and 14.8404))")
    query_payload = sdss.SDSS.query_region(coords_list,
                                           get_query_payload=True,
                                           data_release=dr)
    assert query_payload['cmd'] == expect
    assert query_payload['format'] == 'csv'

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_column_coordinates_payload(dr):
    expect = ("SELECT DISTINCT "
              "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field "
              "FROM PhotoObjAll AS p   WHERE "
              "((p.ra between 2.02291 and 2.02402) and "
              "(p.dec between 14.8393 and 14.8404)) or "
              "((p.ra between 2.02291 and 2.02402) and "
              "(p.dec between 14.8393 and 14.8404))")
    query_payload = sdss.SDSS.query_region(coords_column,
                                           get_query_payload=True,
                                           data_release=dr)
    assert query_payload['cmd'] == expect
    assert query_payload['format'] == 'csv'

def test_field_help_region(patch_get):
    valid_field = sdss.SDSS.query_region(coords, field_help=True)
    assert isinstance(valid_field, dict)
    assert 'photoobj_all' in valid_field

    existing_p_field = sdss.SDSS.query_region(coords,
                                              field_help='psfmag_r')

    existing_p_s_field = sdss.SDSS.query_region(coords,
                                                field_help='psfmag_r')

    non_existing_field = sdss.SDSS.query_region(coords,
                                                field_help='nonexist')
<commit_msg>Fix payload tests to use the monkeypatch when remote-data is off<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from contextlib import contextmanager
import os
import socket
import numpy as np
from numpy.testing import assert_allclose

from astropy.extern import six
from astropy.io import fits
from astropy.table import Column, Table
from astropy.tests.helper import pytest

from ... import sdss
from ...utils.testing_tools import MockResponse
from ...exceptions import TimeoutError
from ...utils import commons

# actual spectra/data are a bit heavy to include in astroquery, so we don't try
# to deal with them.  Would be nice to find a few very small examples

DATA_FILES = {'spectra_id': 'xid_sp.txt',
              'images_id': 'xid_im.txt',
              'spectra': 'emptyfile.fits',
              'images': 'emptyfile.fits'}

@pytest.fixture
def patch_get(request):
    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(sdss.SDSS, '_request', get_mockreturn)
    return mp

@pytest.fixture
def patch_post(request):
    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(sdss.SDSS, '_request', post_mockreturn)
    return mp

@pytest.fixture
def patch_get_slow(request):
    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(sdss.SDSS, '_request', get_mockreturn_slow)
    return mp

@pytest.fixture
def patch_get_readable_fileobj(request):
    @contextmanager
    def get_readable_fileobj_mockreturn(filename, **kwargs):
        file_obj = data_path(DATA_FILES['spectra'])  # TODO: add images option
        encoding = kwargs.get('encoding', None)
        if encoding == 'binary':
            yield open(file_obj, 'rb')
        else:
            yield open(file_obj, 'r', encoding=encoding)

    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(commons, 'get_readable_fileobj',
               get_readable_fileobj_mockreturn)
    return mp

@pytest.fixture
def patch_get_readable_fileobj_slow(request):
    @contextmanager
    def get_readable_fileobj_mockreturn(filename, **kwargs):
        e = six.moves.urllib_error.URLError('timeout')
        e.reason = socket.timeout()
        raise e
        yield True
    mp = request.getfuncargvalue("monkeypatch")
    mp.setattr(commons, 'get_readable_fileobj',
               get_readable_fileobj_mockreturn)
    return mp

def get_mockreturn(method, url, params=None, timeout=10, cache=True, **kwargs):
    if 'SpecObjAll' in params['cmd']:
        filename = data_path(DATA_FILES['spectra_id'])
    else:
        filename = data_path(DATA_FILES['images_id'])
    content = open(filename, 'rb').read()
    return MockResponse(content, **kwargs)

def get_mockreturn_slow(method, url, params=None, timeout=0, **kwargs):
    raise TimeoutError

def post_mockreturn(method, url, params=None, timeout=0, **kwargs):
    filename = data_path(DATA_FILES['images_id'])
    content = open(filename, 'rb').read()
    return MockResponse(content)

def data_path(filename):
    data_dir = os.path.join(os.path.dirname(__file__), 'data')
    return os.path.join(data_dir, filename)

# Test Case: A Seyfert 1 galaxy
coords = commons.ICRSCoordGenerator('0h8m05.63s +14d50m23.3s')

# Test Case: list of coordinates
coords_list = [coords, coords]

# Test Case: Column of coordinates
coords_column = Column(coords_list, name='coordinates')

# We are not testing queries for DR11 because it is not easily available to
# query: "DR11 data are distributed primarily to provide reproducibility of
# published results based on the DR11 data set. As such, not all data-access
# interfaces are supported for DR11."
def url_tester(data_release):
    if data_release < 10:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/search/x_sql.asp'
    if data_release == 10:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/search/x_sql.aspx'
    if data_release >= 12:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/search/x_results.aspx'
    assert sdss.SDSS._last_url == baseurl.format(data_release)

def url_tester_crossid(data_release):
    if data_release < 11:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/crossid/x_crossid.aspx'
    if data_release >= 12:
        baseurl = 'http://skyserver.sdss.org/dr{}/en/tools/search/X_Results.aspx'
    assert sdss.SDSS._last_url == baseurl.format(data_release)

def compare_xid_data(xid, data):
    if six.PY3:
        pytest.xfail('xid/data comparison fails in PY3 because the instrument '
                     'column is bytes in xid and str in data')
    else:
        for col in xid.colnames:
            if xid[col].dtype.type is np.string_:
                assert xid[col] == data[col]
            else:
                assert_allclose(xid[col], data[col])

def image_tester(images, filetype):
    """Test that an image/spectrum is our fake data."""
    assert type(images) == list
    data = fits.open(data_path(DATA_FILES[filetype]))
    assert images[0][0].header == data[0].header
    assert images[0][0].data == data[0].data

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_spectrum(patch_get, patch_get_readable_fileobj, dr,
                       coords=coords):
    xid = sdss.SDSS.query_region(coords, data_release=dr, spectro=True)
    url_tester(dr)
    sp = sdss.SDSS.get_spectra(matches=xid, data_release=dr)
    image_tester(sp, 'spectra')
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_spectrum_mjd(patch_get, patch_get_readable_fileobj, dr):
    sp = sdss.SDSS.get_spectra(plate=2345, fiberID=572, data_release=dr)
    image_tester(sp, 'spectra')

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_spectrum_coords(patch_get, patch_get_readable_fileobj, dr,
                              coords=coords):
    sp = sdss.SDSS.get_spectra(coords, data_release=dr)
    image_tester(sp, 'spectra')

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_sql(patch_get, patch_get_readable_fileobj, dr):
    query = """
            select top 10
              z, ra, dec, bestObjID
            from
              specObj
            where
              class = 'galaxy'
              and z > 0.3
              and zWarning = 0
            """
    xid = sdss.SDSS.query_sql(query, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_image_from_query_region(patch_get, patch_get_readable_fileobj,
                                      dr, coords=coords):
    xid = sdss.SDSS.query_region(coords, data_release=dr)
    # TODO test what img is
    img = sdss.SDSS.get_images(matches=xid)
    image_tester(img, 'images')
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_image_run(patch_get, patch_get_readable_fileobj, dr):
    img = sdss.SDSS.get_images(run=1904, camcol=3, field=164, data_release=dr)
    image_tester(img, 'images')

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_image_coord(patch_get, patch_get_readable_fileobj, dr,
                          coord=coords):
    img = sdss.SDSS.get_images(coords, data_release=dr)
    image_tester(img, 'images')

def test_sdss_template(patch_get, patch_get_readable_fileobj):
    template = sdss.SDSS.get_spectral_template('qso')
    image_tester(template, 'spectra')

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_specobj(patch_get, dr):
    xid = sdss.SDSS.query_specobj(plate=2340, data_release=dr)
    data = Table.read(data_path(DATA_FILES['spectra_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_sdss_photoobj(patch_get, dr):
    xid = sdss.SDSS.query_photoobj(
        run=1904, camcol=3, field=164, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)
    url_tester(dr)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_list_coordinates(patch_get, dr):
    xid = sdss.SDSS.query_region(coords_list, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_column_coordinates(patch_get, dr):
    xid = sdss.SDSS.query_region(coords_column, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    compare_xid_data(xid, data)
    url_tester(dr)

def test_query_timeout(patch_get_slow, coord=coords):
    with pytest.raises(TimeoutError):
        sdss.SDSS.query_region(coords, timeout=1)

def test_spectra_timeout(patch_get, patch_get_readable_fileobj_slow):
    with pytest.raises(TimeoutError):
        sdss.SDSS.get_spectra(plate=2345, fiberID=572)

def test_images_timeout(patch_get, patch_get_readable_fileobj_slow):
    with pytest.raises(TimeoutError):
        sdss.SDSS.get_images(run=1904, camcol=3, field=164)

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_query_crossid(patch_post, dr):
    xid = sdss.SDSS.query_crossid(coords_column, data_release=dr)
    data = Table.read(data_path(DATA_FILES['images_id']),
                      format='ascii.csv', comment='#')
    assert all(xid == data)
    url_tester_crossid(dr)

# ===========
# Payload tests

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_list_coordinates_payload(patch_get, dr):
    expect = ("SELECT DISTINCT "
              "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field "
              "FROM PhotoObjAll AS p   WHERE "
              "((p.ra between 2.02291 and 2.02402) and "
              "(p.dec between 14.8393 and 14.8404)) or "
              "((p.ra between 2.02291 and 2.02402) and "
              "(p.dec between 14.8393 and 14.8404))")
    query_payload = sdss.SDSS.query_region(coords_list,
                                           get_query_payload=True,
                                           data_release=dr)
    assert query_payload['cmd'] == expect
    assert query_payload['format'] == 'csv'

@pytest.mark.parametrize("dr", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12])
def test_column_coordinates_payload(patch_get, dr):
    expect = ("SELECT DISTINCT "
              "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field "
              "FROM PhotoObjAll AS p   WHERE "
              "((p.ra between 2.02291 and 2.02402) and "
              "(p.dec between 14.8393 and 14.8404)) or "
              "((p.ra between 2.02291 and 2.02402) and "
              "(p.dec between 14.8393 and 14.8404))")
    query_payload = sdss.SDSS.query_region(coords_column,
                                           get_query_payload=True,
                                           data_release=dr)
    assert query_payload['cmd'] == expect
    assert query_payload['format'] == 'csv'

def test_field_help_region(patch_get):
    valid_field = sdss.SDSS.query_region(coords, field_help=True)
    assert isinstance(valid_field, dict)
    assert 'photoobj_all' in valid_field

    existing_p_field = sdss.SDSS.query_region(coords,
                                              field_help='psfmag_r')

    existing_p_s_field = sdss.SDSS.query_region(coords,
                                                field_help='psfmag_r')

    non_existing_field = sdss.SDSS.query_region(coords,
                                                field_help='nonexist')
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from impala.catalog import models
from impala.api.v1 import bp
from flask_restful import Api, Resource, abort
from flask import make_response, json

class ApiVersionInfo(Resource):
    # No auth required for this endpoint
    def get(self):
        return {'stable': False}

class ImpalaResource(Resource):
    # TODO require auth
    pass

def all_fields(model):
    columns = [c.name for c in model.__table__.columns]
    return dict([(c, getattr(model, c)) for c in columns])

class Stack(ImpalaResource):
    def get(self, id):
        stack = models.Stack.query.get(id)
        if not stack:
            abort(404, success=False, message="Stack not found")
        return all_fields(stack)

class StackList(ImpalaResource):
    def get(self):
        stacks = models.Stack.query.all()
        return [all_fields(stack) for stack in stacks]

class Format(ImpalaResource):
    def get(self, id):
        format = models.Format.query.get(id)
        if not format:
            abort(404, success=False, message="Format not found")
        return all_fields(format)

class FormatList(ImpalaResource):
    def get(self):
        formats = models.Format.query.all()
        return [all_fields(format) for format in formats]

class HoldingGroup(ImpalaResource):
    def get(self, id):
        holding_group = models.HoldingGroup.query.get(id)
        if not holding_group:
            abort(404, success=False, message="HoldingGroup not found")
        return all_fields(holding_group)

class HoldingGroupList(ImpalaResource):
    def get(self):
        holding_groups = models.HoldingGroup.query.all()
        return [all_fields(holding_group) for holding_group in holding_groups]

class Holding(ImpalaResource):
    def get(self, id):
        holding = models.Holding.query.get(id)
        if not holding:
            abort(404, success=False, message="Holding not found")
        return all_fields(holding)

class HoldingList(ImpalaResource):
    def get(self):
        holdings = models.Holding.query.all()
        return [all_fields(holding) for holding in holdings]

class RotationRelease(ImpalaResource):
    def get(self, id):
        rotation_release = models.RotationRelease.query.get(id)
        if not rotation_release:
            abort(404, success=False, message="RotationRelease not found")
        return all_fields(rotation_release)

class RotationReleaseList(ImpalaResource):
    def get(self):
        rotation_releases = models.RotationRelease.query.all()
        return [all_fields(rotation_release) for rotation_release in rotation_releases]

class HoldingTag(ImpalaResource):
    def get(self, id):
        holding_tag = models.HoldingTag.query.get(id)
        if not holding_tag:
            abort(404, success=False, message="HoldingTag not found")
        return all_fields(holding_tag)

class HoldingTagList(ImpalaResource):
    def get(self):
        holding_tags = models.HoldingTag.query.all()
        return [all_fields(holding_tag) for holding_tag in holding_tags]

class HoldingComment(ImpalaResource):
    def get(self, id):
        holding_comment = models.HoldingComment.query.get(id)
        if not holding_comment:
            abort(404, success=False, message="HoldingComment not found")
        return all_fields(holding_comment)

class HoldingCommentList(ImpalaResource):
    def get(self):
        holding_comments = models.HoldingComment.query.all()
        return [all_fields(holding_comment) for holding_comment in holding_comments]

class Track(ImpalaResource):
    def get(self, id):
        track = models.Track.query.get(id)
        if not track:
            abort(404, success=False, message="Track not found")
        return all_fields(track)

class TrackList(ImpalaResource):
    def get(self):
        tracks = models.Track.query.all()
        return [all_fields(track) for track in tracks]

api = Api(bp)
api.add_resource(ApiVersionInfo, '/')
api.add_resource(Stack, '/stacks/<string:id>')
api.add_resource(StackList, '/stacks')
api.add_resource(Format, '/formats/<string:id>')
api.add_resource(FormatList, '/formats')
api.add_resource(HoldingGroup, '/holding_groups/<string:id>')
api.add_resource(HoldingGroupList, '/holding_groups')
api.add_resource(Holding, '/holdings/<string:id>')
api.add_resource(HoldingList, '/holdings')
api.add_resource(RotationRelease, '/rotaion_releases/<string:id>')
api.add_resource(RotationReleaseList, '/rotaion_releases')
api.add_resource(HoldingTag, '/holding_tags/<string:id>')
api.add_resource(HoldingTagList, '/holding_tags')
api.add_resource(HoldingComment, '/holding_comments/<string:id>')
api.add_resource(HoldingCommentList, '/holding_comments')
api.add_resource(Track, '/tracks/<string:id>')
api.add_resource(TrackList, '/tracks')

@api.representation('application/json')
def output_json(data, code, headers=None):
    resp = make_response(json.dumps(data), code)
    resp.headers.extend(headers or {})
    return resp
<commit_msg>Fix spelling in API endpoints<commit_after>#!/usr/bin/env python3

from impala.catalog import models
from impala.api.v1 import bp
from flask_restful import Api, Resource, abort
from flask import make_response, json

class ApiVersionInfo(Resource):
    # No auth required for this endpoint
    def get(self):
        return {'stable': False}

class ImpalaResource(Resource):
    # TODO require auth
    pass

def all_fields(model):
    columns = [c.name for c in model.__table__.columns]
    return dict([(c, getattr(model, c)) for c in columns])

class Stack(ImpalaResource):
    def get(self, id):
        stack = models.Stack.query.get(id)
        if not stack:
            abort(404, success=False, message="Stack not found")
        return all_fields(stack)

class StackList(ImpalaResource):
    def get(self):
        stacks = models.Stack.query.all()
        return [all_fields(stack) for stack in stacks]

class Format(ImpalaResource):
    def get(self, id):
        format = models.Format.query.get(id)
        if not format:
            abort(404, success=False, message="Format not found")
        return all_fields(format)

class FormatList(ImpalaResource):
    def get(self):
        formats = models.Format.query.all()
        return [all_fields(format) for format in formats]

class HoldingGroup(ImpalaResource):
    def get(self, id):
        holding_group = models.HoldingGroup.query.get(id)
        if not holding_group:
            abort(404, success=False, message="HoldingGroup not found")
        return all_fields(holding_group)

class HoldingGroupList(ImpalaResource):
    def get(self):
        holding_groups = models.HoldingGroup.query.all()
        return [all_fields(holding_group) for holding_group in holding_groups]

class Holding(ImpalaResource):
    def get(self, id):
        holding = models.Holding.query.get(id)
        if not holding:
            abort(404, success=False, message="Holding not found")
        return all_fields(holding)

class HoldingList(ImpalaResource):
    def get(self):
        holdings = models.Holding.query.all()
        return [all_fields(holding) for holding in holdings]

class RotationRelease(ImpalaResource):
    def get(self, id):
        rotation_release = models.RotationRelease.query.get(id)
        if not rotation_release:
            abort(404, success=False, message="RotationRelease not found")
        return all_fields(rotation_release)

class RotationReleaseList(ImpalaResource):
    def get(self):
        rotation_releases = models.RotationRelease.query.all()
        return [all_fields(rotation_release) for rotation_release in rotation_releases]

class HoldingTag(ImpalaResource):
    def get(self, id):
        holding_tag = models.HoldingTag.query.get(id)
        if not holding_tag:
            abort(404, success=False, message="HoldingTag not found")
        return all_fields(holding_tag)

class HoldingTagList(ImpalaResource):
    def get(self):
        holding_tags = models.HoldingTag.query.all()
        return [all_fields(holding_tag) for holding_tag in holding_tags]

class HoldingComment(ImpalaResource):
    def get(self, id):
        holding_comment = models.HoldingComment.query.get(id)
        if not holding_comment:
            abort(404, success=False, message="HoldingComment not found")
        return all_fields(holding_comment)

class HoldingCommentList(ImpalaResource):
    def get(self):
        holding_comments = models.HoldingComment.query.all()
        return [all_fields(holding_comment) for holding_comment in holding_comments]

class Track(ImpalaResource):
    def get(self, id):
        track = models.Track.query.get(id)
        if not track:
            abort(404, success=False, message="Track not found")
        return all_fields(track)

class TrackList(ImpalaResource):
    def get(self):
        tracks = models.Track.query.all()
        return [all_fields(track) for track in tracks]

api = Api(bp)
api.add_resource(ApiVersionInfo, '/')
api.add_resource(Stack, '/stacks/<string:id>')
api.add_resource(StackList, '/stacks')
api.add_resource(Format, '/formats/<string:id>')
api.add_resource(FormatList, '/formats')
api.add_resource(HoldingGroup, '/holding_groups/<string:id>')
api.add_resource(HoldingGroupList, '/holding_groups')
api.add_resource(Holding, '/holdings/<string:id>')
api.add_resource(HoldingList, '/holdings')
api.add_resource(RotationRelease, '/rotation_releases/<string:id>')
api.add_resource(RotationReleaseList, '/rotation_releases')
api.add_resource(HoldingTag, '/holding_tags/<string:id>')
api.add_resource(HoldingTagList, '/holding_tags')
api.add_resource(HoldingComment, '/holding_comments/<string:id>')
api.add_resource(HoldingCommentList, '/holding_comments')
api.add_resource(Track, '/tracks/<string:id>')
api.add_resource(TrackList, '/tracks')

@api.representation('application/json')
def output_json(data, code, headers=None):
    resp = make_response(json.dumps(data), code)
    resp.headers.extend(headers or {})
    return resp
<|endoftext|>


<commit_before>try:
    from setuptools import setup
except ImportError as err:
    print("Missing Python module requirement: setuptools.")
    raise err

from tabpy_server import __version__

setup(
    name='tabpy-server',
    version=__version__,
    description='Web server Tableau uses to run Python scripts.',
    url='https://github.com/tableau/TabPy',
    author='Tableau',
    author_email='github@tableau.com',
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Programming Language :: Python :: 3.6',
        "Topic :: Scientific/Engineering",
        "Topic :: Scientific/Engineering :: Information Analysis",
    ],
    packages=['tabpy_server',
              'tabpy_server.common',
              'tabpy_server.management',
              'tabpy_server.psws',
              'tabpy_server.static'],
    package_data={'tabpy_server.static': ['*.*'],
                  'tabpy_server': ['startup.*', 'state.ini']},
    license='MIT',
# Note: many of these required packages are included in base python
# but are listed here because different linux distros use custom
# python installations.  And users can remove packages at any point
    install_requires=[
        'backports_abc',
        'cloudpickle',
        'configparser',
        'decorator',
        'future',
        'genson',
        'jsonschema~=2.3.0',
        'mock',
        'numpy',
        'pyopenssl',
        'python-dateutil',
        'requests',
        'simplejson',
        'singledispatch',
        'six',
        'tornado==5.1.1',
        'Tornado-JSON',
        'urllib3'
    ]
)
<commit_msg>Update setup.py<commit_after>try:
    from setuptools import setup
except ImportError as err:
    print("Missing Python module requirement: setuptools.")
    raise err

from tabpy_server import __version__

setup(
    name='tabpy-server',
    version=__version__,
    description='Web server Tableau uses to run Python scripts.',
    url='https://github.com/tableau/TabPy',
    author='Tableau',
    author_email='github@tableau.com',
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Programming Language :: Python :: 3.6',
        "Topic :: Scientific/Engineering",
        "Topic :: Scientific/Engineering :: Information Analysis",
    ],
    packages=['tabpy_server',
              'tabpy_server.common',
              'tabpy_server.management',
              'tabpy_server.psws',
              'tabpy_server.static'],
    package_data={'tabpy_server.static': ['*.*'],
                  'tabpy_server': ['startup.*', 'state.ini']},
    license='MIT',
    # Note: many of these required packages are included in base python
    # but are listed here because different linux distros use custom
    # python installations.  And users can remove packages at any point
    install_requires=[
        'backports_abc',
        'cloudpickle',
        'configparser',
        'decorator',
        'future',
        'genson',
        'jsonschema~=2.3.0',
        'mock',
        'numpy',
        'pyopenssl',
        'python-dateutil',
        'requests',
        'simplejson',
        'singledispatch',
        'six',
        'tornado==5.1.1',
        'Tornado-JSON',
        'urllib3'
    ]
)
<|endoftext|>


<commit_before># Copyright 2014 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for compiling Python functions into Impala UDFs"""

from __future__ import absolute_import

import llvm.core as lc
from numba import sigutils
from numba.compiler import compile_extra, Flags

from .target import ImpalaTargetContext
from .typing import impala_typing_context
from .types import (FunctionContext, BooleanVal, TinyIntVal, SmallIntVal,
                    IntVal, BigIntVal, FloatVal, DoubleVal, StringVal)

# functionality to compile Python UDFs into Impala-executable IR

def udf(signature):
    def wrapper(pyfunc):
        udfobj = UDF(pyfunc, signature)
        return udfobj
    return wrapper

class UDF(object):
    def __init__(self, pyfunc, signature):
        self.py_func = pyfunc
        self.signature = signature
        self.name = pyfunc.__name__

        # recreate for each UDF, as linking is destructive to the
        # precompiled module
        impala_typing = impala_typing_context()
        impala_targets = ImpalaTargetContext(impala_typing)

        args, return_type = sigutils.normalize_signature(signature)
        flags = Flags()
        flags.set('no_compile')
        self._cres = compile_extra(typingctx=impala_typing,
                   targetctx=impala_targets, func=pyfunc,
                   args=args, return_type=return_type,
                   flags=flags, locals={})
        llvm_func = impala_targets.finalize(self._cres.llvm_func, return_type,
                    args)
        self.llvm_func = llvm_func
        numba_module = llvm_func.module
        self.llvm_module = lc.Module.new(self.name)
        self.llvm_module.link_in(numba_module)
        self.llvm_module.link_in(impala_targets.precompiled_module)

# functionality to ship code to Impala cluster

# convert *Val name to Impala data type name
udf2impala_type = {'BooleanVal': 'BOOLEAN',
                   'TinyIntVal': 'TINYINT',
                   'SmallIntVal': 'SMALLINT',
                   'IntVal': 'INT',
                   'BigIntVal': 'BIGINT',
                   'FloatVal': 'FLOAT',
                   'DoubleVal': 'DOUBLE',
                   'StringVal': 'STRING',
                   'TimestampVal': 'TIMESTAMP'}

# TODO: in the future, consider taking an "ImpalaContext" if there is some info I
# want to store.  But this could also all be in the cursor object potentially
try:
    from pywebhdfs.webhdfs import PyWebHdfsClient

    def ship_udf(cursor, function, hdfs_path, host, webhdfs_port=50070, user=None,
            udf_name=None, overwrite=False):
        # extract some information from the function
        if udf_name is None:
            udf_name = function.name
        symbol = function.llvm_func.name
        ir = function.llvm_module.to_bitcode()
        return_type = udf2impala_type[function.signature.return_type.name]
        arg_types = [udf2impala_type[arg.name] for arg in function.signature.args[1:]]

        # ship the IR to the cluster
        hdfs_client = PyWebHdfsClient(host=host, port=webhdfs_port, user_name=user)
        if not hdfs_path.endswith('.ll'):
            raise ValueError("The HDFS file name must end with .ll")
        hdfs_client.create_file(hdfs_path.lstrip('/'), ir, overwrite=overwrite)

        # register the function in Impala
        impala_name = '%s(%s)' % (udf_name, ', '.join(arg_types))
        if overwrite:
            cursor.execute("SHOW FUNCTIONS")
            registered_functions = [fn[0].lower() for fn in cursor.fetchall()]
            if impala_name.lower() in registered_functions:
                cursor.execute("DROP FUNCTION %s" % impala_name)
        register_query = "CREATE FUNCTION %s RETURNS %s LOCATION '%s' SYMBOL='%s'" % (impala_name,
                return_type, hdfs_path, symbol)
        cursor.execute(register_query)

except ImportError:
    print "Failed to import pywebhdfs; you must ship your Python UDFs manually."
<commit_msg>Updated ship_udf() to use ImpalaContext etc.<commit_after># Copyright 2014 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for compiling Python functions into Impala UDFs"""

from __future__ import absolute_import

import os

import llvm.core as lc
from numba import sigutils
from numba.compiler import compile_extra, Flags

from impala.udf.target import ImpalaTargetContext
from impala.udf.typing import impala_typing_context
from impala.udf.types import (BooleanVal, TinyIntVal, SmallIntVal, IntVal,
        BigIntVal, FloatVal, DoubleVal, StringVal)

# functionality to compile Python UDFs into Impala-executable IR

def udf(signature):
    def wrapper(pyfunc):
        udfobj = UDF(pyfunc, signature)
        return udfobj
    return wrapper

class UDF(object):
    def __init__(self, pyfunc, signature):
        self.py_func = pyfunc
        self.signature = signature
        self.name = pyfunc.__name__

        # recreate for each UDF, as linking is destructive to the
        # precompiled module
        impala_typing = impala_typing_context()
        impala_targets = ImpalaTargetContext(impala_typing)

        args, return_type = sigutils.normalize_signature(signature)
        flags = Flags()
        flags.set('no_compile')
        self._cres = compile_extra(typingctx=impala_typing,
                   targetctx=impala_targets, func=pyfunc,
                   args=args, return_type=return_type,
                   flags=flags, locals={})
        llvm_func = impala_targets.finalize(self._cres.llvm_func, return_type,
                    args)
        self.llvm_func = llvm_func
        numba_module = llvm_func.module
        self.llvm_module = lc.Module.new(self.name)
        self.llvm_module.link_in(numba_module)
        self.llvm_module.link_in(impala_targets.precompiled_module)

# functionality to ship code to Impala cluster

# convert *Val name to Impala data type name
udf_to_impala_type = {'BooleanVal': 'BOOLEAN',
                      'TinyIntVal': 'TINYINT',
                      'SmallIntVal': 'SMALLINT',
                      'IntVal': 'INT',
                      'BigIntVal': 'BIGINT',
                      'FloatVal': 'FLOAT',
                      'DoubleVal': 'DOUBLE',
                      'StringVal': 'STRING',
                      'TimestampVal': 'TIMESTAMP'}

# TODO: in the future, consider taking an "ImpalaContext" if there is some info I
# want to store.  But this could also all be in the cursor object potentially
try:
    from pywebhdfs.webhdfs import PyWebHdfsClient

    def ship_udf(ic, function, hdfs_path=None, udf_name=None, database=None,
            overwrite=False):
        # extract some information from the function
        if udf_name is None:
            udf_name = function.name
        symbol = function.llvm_func.name
        ir = function.llvm_module.to_bitcode()
        return_type = udf_to_impala_type[function.signature.return_type.name]
        arg_types = [udf_to_impala_type[arg.name]
                        for arg in function.signature.args[1:]]

        # ship the IR to the cluster
        hdfs_client = PyWebHdfsClient(host=ic._nn_host, port=ic._webhdfs_port,
                user_name=ic._hdfs_user)
        if hdfs_path is None:
            hdfs_path = os.path.join(ic._temp_dir, udf_name + '.ll')
        if not hdfs_path.endswith('.ll'):
            raise ValueError("The HDFS file name must end with .ll")
        hdfs_client.create_file(hdfs_path.lstrip('/'), ir, overwrite=overwrite)

        # register the function in Impala
        if database is None:
            database = ic._temp_db
        impala_name = '%s.%s(%s)' % (database, udf_name, ', '.join(arg_types))
        if overwrite:
            ic._cursor.execute("DROP FUNCTION IF EXISTS %s" % impala_name)
        register_query = "CREATE FUNCTION %s RETURNS %s LOCATION '%s' SYMBOL='%s'" % (impala_name,
                return_type, hdfs_path, symbol)
        ic._cursor.execute(register_query)

except ImportError:
    print "Failed to import pywebhdfs; you must ship your Python UDFs manually."
<|endoftext|>


<commit_before># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import os
import csv
import numpy

from nose.plugins.attrib import attr

from qa_tests import risk
from tests.utils import helpers

from openquake.db import models

class ScenarioDamageRiskCase1TestCase(risk.BaseRiskQATestCase):
    cfg = os.path.join(os.path.dirname(__file__), 'job.ini')

    @attr('qa', 'risk', 'scenario_damage')
    def test(self):
        self._run_test()

    def hazard_id(self):
        job = helpers.get_hazard_job(
            helpers.demo_file("scenario_hazard/job.ini"))

        job.hazard_calculation = models.HazardCalculation.objects.create(
            owner=job.hazard_calculation.owner,
            truncation_level=job.hazard_calculation.truncation_level,
            maximum_distance=job.hazard_calculation.maximum_distance,
            intensity_measure_types_and_levels=(
                job.hazard_calculation.intensity_measure_types_and_levels),
            calculation_mode="scenario")

        job.save()

        output = models.Output.objects.create_output(
            job, "Test Hazard output", "gmf_scenario")

        fname = os.path.join(os.path.dirname(__file__), 'gmf_scenario.csv')
        with open(fname, 'rb') as csvfile:
            gmfreader = csv.reader(csvfile, delimiter=',')
            locations = gmfreader.next()

            arr = numpy.array([map(float, row) for row in gmfreader])
            for i, gmvs in enumerate(arr.transpose()):
                models.GmfScenario.objects.create(
                    output=output,
                    imt="MMI",
                    gmvs=gmvs,
                    result_grp_ordinal=1,
                    location="POINT(%s)" % locations[i])

        return output.id

    def actual_data(self, job):
        pass

    def expected_data(self):
        pass

    def expected_outputs(self):
        return []
<commit_msg>Temporary change to let pass an empty test<commit_after># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import os
import csv
import numpy

from nose.plugins.attrib import attr

from qa_tests import risk
from tests.utils import helpers

from openquake.db import models

class ScenarioDamageRiskCase1TestCase(risk.BaseRiskQATestCase):
    cfg = os.path.join(os.path.dirname(__file__), 'job.ini')

    @attr('qa', 'risk', 'scenario_damage')
    def test(self):
        self._run_test()

    def hazard_id(self):
        job = helpers.get_hazard_job(
            helpers.demo_file("scenario_hazard/job.ini"))

        job.hazard_calculation = models.HazardCalculation.objects.create(
            owner=job.hazard_calculation.owner,
            truncation_level=job.hazard_calculation.truncation_level,
            maximum_distance=job.hazard_calculation.maximum_distance,
            intensity_measure_types_and_levels=(
                job.hazard_calculation.intensity_measure_types_and_levels),
            calculation_mode="scenario")

        job.save()

        output = models.Output.objects.create_output(
            job, "Test Hazard output", "gmf_scenario")

        fname = os.path.join(os.path.dirname(__file__), 'gmf_scenario.csv')
        with open(fname, 'rb') as csvfile:
            gmfreader = csv.reader(csvfile, delimiter=',')
            locations = gmfreader.next()

            arr = numpy.array([map(float, row) for row in gmfreader])
            for i, gmvs in enumerate(arr.transpose()):
                models.GmfScenario.objects.create(
                    output=output,
                    imt="MMI",
                    gmvs=gmvs,
                    result_grp_ordinal=1,
                    location="POINT(%s)" % locations[i])

        return output.id

    def actual_data(self, job):
        return []

    def expected_data(self):
        return []

    def expected_outputs(self):
        return []
<|endoftext|>


<commit_before><commit_msg>Add testing<commit_after><|endoftext|>


<commit_before>"""
Django settings for NaschpunkteDP project.

Generated by 'django-admin startproject' using Django 1.11.2.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'uk6juk*tmfx1ahk-428zc7#p)mz*dlz%iwos92p#jrs!r#v9iy'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'NaschpunkteApp',
]

MIDDLEWARE = [
    'google.appengine.ext.ndb.django_middleware.NdbDjangoMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'NaschpunkteDP.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'NaschpunkteDP.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    #'default': {
    #    'ENGINE': 'django.db.backends.sqlite3',
    #    'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    #}
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_URL = '/static/'
<commit_msg>fix settings<commit_after>"""
Django settings for NaschpunkteDP project.

Generated by 'django-admin startproject' using Django 1.11.2.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'uk6juk*tmfx1ahk-428zc7#p)mz*dlz%iwos92p#jrs!r#v9iy'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'NaschpunkteApp',
]

MIDDLEWARE_CLASSES = [
    'google.appengine.ext.ndb.django_middleware.NdbDjangoMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'NaschpunkteDP.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'NaschpunkteDP.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    #'default': {
    #    'ENGINE': 'django.db.backends.sqlite3',
    #    'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    #}
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_URL = '/static/'
<|endoftext|>