<commit_before>import operator
import os
import pickle
import re
import threading
from collections import deque, namedtuple
from functools import reduce

import numpy as np
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _

from c3nav.mapdata.models import MapUpdate, Space
from c3nav.mapdata.utils.locations import CustomLocation
from c3nav.routing.router import Router

class Locator:
    filename = os.path.join(settings.CACHE_ROOT, 'locator')

    def __init__(self, stations, spaces):
        self.stations = stations
        self.spaces = spaces

    @classmethod
    def rebuild(cls, update):
        stations = LocatorStations()
        spaces = {}
        for space in Space.objects.prefetch_related('wifi_measurements'):
            new_space = LocatorSpace(
                pk=space.pk,
                points=(LocatorPoint.from_measurement(measurement, stations)
                        for measurement in space.wifi_measurements.all())
            )
            if new_space.points:
                spaces[space.pk] = new_space

        locator = cls(stations, spaces)
        pickle.dump(locator, open(cls.build_filename(update), 'wb'))
        return locator

    @classmethod
    def build_filename(cls, update):
        return os.path.join(settings.CACHE_ROOT, 'locator_%s.pickle' % MapUpdate.build_cache_key(*update))

    @classmethod
    def load_nocache(cls, update):
        return pickle.load(open(cls.build_filename(update), 'rb'))

    cached = None
    cache_update = None
    cache_lock = threading.Lock()

    @classmethod
    def load(cls):
        from c3nav.mapdata.models import MapUpdate
        update = MapUpdate.last_processed_update()
        if cls.cache_update != update:
            with cls.cache_lock:
                cls.cache_update = update
                cls.cached = cls.load_nocache(update)
        return cls.cached

    def locate(self, scan, permissions=None):
        router = Router.load()
        restrictions = router.get_restrictions(permissions)

        scan = LocatorPoint.clean_scan(scan, ignore_invalid_stations=True)
        scan_values = LocatorPoint.convert_scan(scan, self.stations, create=False)
        if not scan_values:
            return None

        # convert scan values
        scan_values = {station_id: value**2 for station_id, value in scan_values.items()}

        # get visible spaces
        spaces = tuple(space for pk, space in self.spaces.items() if pk not in restrictions.spaces)

        # find best point
        best_station_id = max(scan_values.items(), key=operator.itemgetter(1))[0]
        best_location = None
        best_score = float('inf')
        for space in spaces:
            point, score = space.get_best_point(scan_values, needed_station_id=best_station_id)
            if point is None:
                continue
            if score < best_score:
                location = CustomLocation(router.spaces[space.pk].level, point.x, point.y,
                                          permissions=permissions, icon='my_location')
                best_location = location
                best_score = score

        return best_location

class LocatorStations:
    def __init__(self):
        self.stations = []
        self.stations_lookup = {}

    def get(self, bssid, ssid, frequency, create=False):
        station_id = self.stations_lookup.get((bssid, None), None)
        if station_id is not None:
            station = self.stations[station_id]
            station.frequencies.add(frequency)
        elif create:
            station = LocatorStation(bssid, ssid, set((frequency, )))
            station_id = len(self.stations)
            self.stations_lookup[(bssid, None)] = station_id
            self.stations.append(station)
        return station_id

class LocatorSpace:
    no_signal = int(-90)**2

    def __init__(self, pk, points):
        self.pk = pk
        self.points = tuple(points)
        self.stations_set = reduce(operator.or_, (frozenset(point.values.keys()) for point in self.points), frozenset())
        self.stations = tuple(self.stations_set)
        self.stations_lookup = {station_id: i for i, station_id in enumerate(self.stations)}

        self.levels = np.full((len(self.points), len(self.stations)), fill_value=self.no_signal, dtype=np.int64)
        for i, point in enumerate(self.points):
            for station_id, value in point.values.items():
                self.levels[i][self.stations_lookup[station_id]] = int(value)**2

    def get_best_point(self, scan_values, needed_station_id=None):
        # check if this space knows the needed station id, otherwise no results here
        if needed_station_id not in self.stations_set:
            return None, None

        # stations that this space knows
        station_ids = frozenset(scan_values.keys()) & self.stations_set
        penalty = 0
        for station_id, value in scan_values.items():
            if station_id not in self.stations_set:
                penalty += (value - self.no_signal)**2

        stations = tuple(self.stations_lookup[station_id] for station_id in station_ids)
        values = np.array(tuple(scan_values[station_id] for station_id in station_ids), dtype=np.int64)

        # acceptable points need to have a value for the needed_station_id
        points = tuple(
            np.argwhere(self.levels[:, self.stations_lookup[needed_station_id]] < self.no_signal).ravel()
        )
        if not points:
            return None, None

        scores = (np.sum(
            (self.levels[np.array(points, dtype=np.uint32).reshape((-1, 1)), stations] - values)**2,
            axis=1
        )+penalty) / len(scan_values)
        best_point_i = np.argmin(scores).ravel()[0]
        best_point = points[best_point_i]
        return self.points[best_point], scores[best_point_i]

class LocatorPoint(namedtuple('LocatorPoint', ('x', 'y', 'values'))):
    @classmethod
    def from_measurement(cls, measurement, stations: LocatorStations):
        return cls(x=measurement.geometry.x, y=measurement.geometry.y,
                   values=cls.convert_scans(measurement.data, stations, create=True))

    @classmethod
    def convert_scan(cls, scan, stations: LocatorStations, create=False):
        values = {}
        for scan_value in scan:
            if settings.WIFI_SSIDS and scan_value['ssid'] not in settings.WIFI_SSIDS:
                continue
            station_id = stations.get(bssid=scan_value['bssid'], ssid=scan_value['ssid'],
                                      frequency=scan_value['frequency'], create=create)
            if station_id is not None:
                # todo: convert to something more or less linear
                values[station_id] = scan_value['level']
        return values

    @classmethod
    def convert_scans(cls, scans, stations: LocatorStations, create=False):
        values_list = deque()
        for scan in scans:
            values_list.append(cls.convert_scan(scan, stations, create))

        station_ids = reduce(operator.or_, (frozenset(values.keys()) for values in values_list), frozenset())
        return {
            station_id: cls.average(tuple(values[station_id] for values in values_list if station_id in values))
            for station_id in station_ids
        }

    @staticmethod
    def average(items):
        return sum(items) / len(items)

    valid_frequencies = frozenset((
        2412, 2417, 2422, 2427, 2432, 2437, 2442, 2447, 2452, 2457, 2462, 2467, 2472, 2484,
        5180, 5190, 5200, 5210, 5220, 5230, 5240, 5250, 5260, 5270, 5280, 5290, 5300, 5310, 5320,
        5500, 5510, 5520, 5530, 5540, 5550, 5560, 5570, 5580, 5590, 5600, 5610, 5620, 5630, 5640,
        5660, 5670, 5680, 5690, 5700, 5710, 5720, 5745, 5755, 5765, 5775, 5785, 5795, 5805, 5825
    ))
    needed_keys = frozenset(('bssid', 'ssid', 'level', 'frequency'))
    allowed_keys = needed_keys | frozenset(('last', ))

    @classmethod
    def clean_scans(cls, data, ignore_invalid_stations=False):
        if not isinstance(data, list):
            raise ValidationError(_('Invalid Scan. Scans list list not a list.'))
        return tuple(cls.clean_scan(scan) for scan in data)

    @classmethod
    def clean_scan(cls, data, ignore_invalid_stations=False):
        if not isinstance(data, list):
            raise ValidationError(_('Invalid Scan. Scan not a list.'))
        cleaned_scan = deque()
        for scan_value in data:
            try:
                cleaned_scan.append(cls.clean_scan_value(scan_value))
            except ValidationError:
                if not ignore_invalid_stations:
                    raise
        return tuple(cleaned_scan)

    @classmethod
    def clean_scan_value(cls, data):
        if not isinstance(data, dict):
            raise ValidationError(_('Invalid Scan. Scan value not a dictionary.'))
        keys = frozenset(data.keys())
        if (keys - cls.allowed_keys) or (cls.needed_keys - keys):
            raise ValidationError(_('Invalid Scan. Missing or forbidden keys.'))
        if not isinstance(data['bssid'], str):
            raise ValidationError(_('Invalid Scan. BSSID not a String.'))
        data['bssid'] = data['bssid'].upper()
        if not re.match(r'^([0-9A-F]{2}:){5}[0-9A-F]{2}$', data['bssid']):
            raise ValidationError(_('Invalid Scan. Invalid BSSID.'))
        if not isinstance(data['level'], int) or not (-1 >= data['level'] >= -100):
            raise ValidationError(_('Invalid Scan. Invalid RSSI/Level.'))
        if data['frequency'] not in cls.valid_frequencies:
            raise ValidationError(_('Invalid Scan. Not an allowed frequency.'))
        if 'last' in keys and (not isinstance(data['last'], int) or data['last'] <= 0):
            raise ValidationError(_('Invalid Scan. Invalid last timestamp.'))
        return data

class LocatorStation:
    def __init__(self, bssid, ssid, frequencies=()):
        self.bssid = bssid
        self.ssid = ssid
        self.frequencies = set(frequencies)

    def __repr__(self):
        return 'LocatorStation(%r, %r, frequencies=%r)' % (self.bssid, self.ssid, self.frequencies)
<commit_msg>add helpful comment<commit_after>import operator
import os
import pickle
import re
import threading
from collections import deque, namedtuple
from functools import reduce

import numpy as np
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _

from c3nav.mapdata.models import MapUpdate, Space
from c3nav.mapdata.utils.locations import CustomLocation
from c3nav.routing.router import Router

class Locator:
    filename = os.path.join(settings.CACHE_ROOT, 'locator')

    def __init__(self, stations, spaces):
        self.stations = stations
        self.spaces = spaces

    @classmethod
    def rebuild(cls, update):
        stations = LocatorStations()
        spaces = {}
        for space in Space.objects.prefetch_related('wifi_measurements'):
            new_space = LocatorSpace(
                pk=space.pk,
                points=(LocatorPoint.from_measurement(measurement, stations)
                        for measurement in space.wifi_measurements.all())
            )
            if new_space.points:
                spaces[space.pk] = new_space

        locator = cls(stations, spaces)
        pickle.dump(locator, open(cls.build_filename(update), 'wb'))
        return locator

    @classmethod
    def build_filename(cls, update):
        return os.path.join(settings.CACHE_ROOT, 'locator_%s.pickle' % MapUpdate.build_cache_key(*update))

    @classmethod
    def load_nocache(cls, update):
        return pickle.load(open(cls.build_filename(update), 'rb'))

    cached = None
    cache_update = None
    cache_lock = threading.Lock()

    @classmethod
    def load(cls):
        from c3nav.mapdata.models import MapUpdate
        update = MapUpdate.last_processed_update()
        if cls.cache_update != update:
            with cls.cache_lock:
                cls.cache_update = update
                cls.cached = cls.load_nocache(update)
        return cls.cached

    def locate(self, scan, permissions=None):
        router = Router.load()
        restrictions = router.get_restrictions(permissions)

        scan = LocatorPoint.clean_scan(scan, ignore_invalid_stations=True)
        scan_values = LocatorPoint.convert_scan(scan, self.stations, create=False)
        if not scan_values:
            return None

        # convert scan values
        scan_values = {station_id: value**2 for station_id, value in scan_values.items()}

        # get visible spaces
        spaces = tuple(space for pk, space in self.spaces.items() if pk not in restrictions.spaces)

        # find best point
        best_station_id = max(scan_values.items(), key=operator.itemgetter(1))[0]
        best_location = None
        best_score = float('inf')
        for space in spaces:
            point, score = space.get_best_point(scan_values, needed_station_id=best_station_id)
            if point is None:
                continue
            if score < best_score:
                location = CustomLocation(router.spaces[space.pk].level, point.x, point.y,
                                          permissions=permissions, icon='my_location')
                best_location = location
                best_score = score

        return best_location

class LocatorStations:
    def __init__(self):
        self.stations = []
        self.stations_lookup = {}

    def get(self, bssid, ssid, frequency, create=False):
        # yes, we're not looking up frequencies, because they can change… yep…
        station_id = self.stations_lookup.get((bssid, None), None)
        if station_id is not None:
            station = self.stations[station_id]
            station.frequencies.add(frequency)
        elif create:
            station = LocatorStation(bssid, ssid, set((frequency, )))
            station_id = len(self.stations)
            self.stations_lookup[(bssid, None)] = station_id
            self.stations.append(station)
        return station_id

class LocatorSpace:
    no_signal = int(-90)**2

    def __init__(self, pk, points):
        self.pk = pk
        self.points = tuple(points)
        self.stations_set = reduce(operator.or_, (frozenset(point.values.keys()) for point in self.points), frozenset())
        self.stations = tuple(self.stations_set)
        self.stations_lookup = {station_id: i for i, station_id in enumerate(self.stations)}

        self.levels = np.full((len(self.points), len(self.stations)), fill_value=self.no_signal, dtype=np.int64)
        for i, point in enumerate(self.points):
            for station_id, value in point.values.items():
                self.levels[i][self.stations_lookup[station_id]] = int(value)**2

    def get_best_point(self, scan_values, needed_station_id=None):
        # check if this space knows the needed station id, otherwise no results here
        if needed_station_id not in self.stations_set:
            return None, None

        # stations that this space knows
        station_ids = frozenset(scan_values.keys()) & self.stations_set
        penalty = 0
        for station_id, value in scan_values.items():
            if station_id not in self.stations_set:
                penalty += (value - self.no_signal)**2

        stations = tuple(self.stations_lookup[station_id] for station_id in station_ids)
        values = np.array(tuple(scan_values[station_id] for station_id in station_ids), dtype=np.int64)

        # acceptable points need to have a value for the needed_station_id
        points = tuple(
            np.argwhere(self.levels[:, self.stations_lookup[needed_station_id]] < self.no_signal).ravel()
        )
        if not points:
            return None, None

        scores = (np.sum(
            (self.levels[np.array(points, dtype=np.uint32).reshape((-1, 1)), stations] - values)**2,
            axis=1
        )+penalty) / len(scan_values)
        best_point_i = np.argmin(scores).ravel()[0]
        best_point = points[best_point_i]
        return self.points[best_point], scores[best_point_i]

class LocatorPoint(namedtuple('LocatorPoint', ('x', 'y', 'values'))):
    @classmethod
    def from_measurement(cls, measurement, stations: LocatorStations):
        return cls(x=measurement.geometry.x, y=measurement.geometry.y,
                   values=cls.convert_scans(measurement.data, stations, create=True))

    @classmethod
    def convert_scan(cls, scan, stations: LocatorStations, create=False):
        values = {}
        for scan_value in scan:
            if settings.WIFI_SSIDS and scan_value['ssid'] not in settings.WIFI_SSIDS:
                continue
            station_id = stations.get(bssid=scan_value['bssid'], ssid=scan_value['ssid'],
                                      frequency=scan_value['frequency'], create=create)
            if station_id is not None:
                # todo: convert to something more or less linear
                values[station_id] = scan_value['level']
        return values

    @classmethod
    def convert_scans(cls, scans, stations: LocatorStations, create=False):
        values_list = deque()
        for scan in scans:
            values_list.append(cls.convert_scan(scan, stations, create))

        station_ids = reduce(operator.or_, (frozenset(values.keys()) for values in values_list), frozenset())
        return {
            station_id: cls.average(tuple(values[station_id] for values in values_list if station_id in values))
            for station_id in station_ids
        }

    @staticmethod
    def average(items):
        return sum(items) / len(items)

    valid_frequencies = frozenset((
        2412, 2417, 2422, 2427, 2432, 2437, 2442, 2447, 2452, 2457, 2462, 2467, 2472, 2484,
        5180, 5190, 5200, 5210, 5220, 5230, 5240, 5250, 5260, 5270, 5280, 5290, 5300, 5310, 5320,
        5500, 5510, 5520, 5530, 5540, 5550, 5560, 5570, 5580, 5590, 5600, 5610, 5620, 5630, 5640,
        5660, 5670, 5680, 5690, 5700, 5710, 5720, 5745, 5755, 5765, 5775, 5785, 5795, 5805, 5825
    ))
    needed_keys = frozenset(('bssid', 'ssid', 'level', 'frequency'))
    allowed_keys = needed_keys | frozenset(('last', ))

    @classmethod
    def clean_scans(cls, data, ignore_invalid_stations=False):
        if not isinstance(data, list):
            raise ValidationError(_('Invalid Scan. Scans list list not a list.'))
        return tuple(cls.clean_scan(scan) for scan in data)

    @classmethod
    def clean_scan(cls, data, ignore_invalid_stations=False):
        if not isinstance(data, list):
            raise ValidationError(_('Invalid Scan. Scan not a list.'))
        cleaned_scan = deque()
        for scan_value in data:
            try:
                cleaned_scan.append(cls.clean_scan_value(scan_value))
            except ValidationError:
                if not ignore_invalid_stations:
                    raise
        return tuple(cleaned_scan)

    @classmethod
    def clean_scan_value(cls, data):
        if not isinstance(data, dict):
            raise ValidationError(_('Invalid Scan. Scan value not a dictionary.'))
        keys = frozenset(data.keys())
        if (keys - cls.allowed_keys) or (cls.needed_keys - keys):
            raise ValidationError(_('Invalid Scan. Missing or forbidden keys.'))
        if not isinstance(data['bssid'], str):
            raise ValidationError(_('Invalid Scan. BSSID not a String.'))
        data['bssid'] = data['bssid'].upper()
        if not re.match(r'^([0-9A-F]{2}:){5}[0-9A-F]{2}$', data['bssid']):
            raise ValidationError(_('Invalid Scan. Invalid BSSID.'))
        if not isinstance(data['level'], int) or not (-1 >= data['level'] >= -100):
            raise ValidationError(_('Invalid Scan. Invalid RSSI/Level.'))
        if data['frequency'] not in cls.valid_frequencies:
            raise ValidationError(_('Invalid Scan. Not an allowed frequency.'))
        if 'last' in keys and (not isinstance(data['last'], int) or data['last'] <= 0):
            raise ValidationError(_('Invalid Scan. Invalid last timestamp.'))
        return data

class LocatorStation:
    def __init__(self, bssid, ssid, frequencies=()):
        self.bssid = bssid
        self.ssid = ssid
        self.frequencies = set(frequencies)

    def __repr__(self):
        return 'LocatorStation(%r, %r, frequencies=%r)' % (self.bssid, self.ssid, self.frequencies)
<|endoftext|>


<commit_before>"""Routes configuration

The more specific and detailed routes should be defined first so they
may take precedent over the more generic routes. For more information
refer to the routes manual at http://routes.groovie.org/docs/
"""
from pylons import config
from routes import Mapper

wiki = 'http://wiki.pylonshq.com'

def article_expand(kwargs):
    if 'article' not in kwargs:
        return kwargs
    article = kwargs.pop('article')
    kwargs['year'] = article.published.year
    kwargs['month'] = article.published.month
    kwargs['slug'] = article.slug
    return kwargs

def make_map(globs=None):
    """Create, configure and return the routes Mapper"""
    version = config['pylons.app_globals'].current_version
    static_host = config['cdn.uri']

    map = Mapper(directory=config['pylons.paths']['controllers'],
                 always_scan=config['debug'])
    map.minimization = False

    # The ErrorController route (handles 404/500 error pages); it should
    # likely stay at the top, ensuring it can always be resolved
    map.connect('/error/{action}', controller='error')
    map.connect('/error/{action}/{id}', controller='error')

    # Home url's
    map.connect('home', '/', controller='home', action='index')
    map.connect('features', '/features', controller='home', action='features')
    map.connect('history', '/history', controller='home', action='history')

    # Blog url's
    map.connect('article_archives', '/articles/archives/{year:\d+}/{month:\d+}/{slug}',
                controller='articles', action='archives', _filter=article_expand)
    map.resource('article', 'articles')

    # Code url's
    map.connect('buildbot', '/buildbot/{action}', controller='buildbot')

    # Doc url's
    map.connect('doc_upload', '/docs/upload', controller='docs', action='upload')
    map.connect('doc_upload_image', '/docs/upload_image', controller='docs', action='upload_image')
    map.connect('doc_delete', '/docs/delete_revision/{project}/{version}', controller='docs', action='delete_revision')
    map.redirect('/docs/{version}', '/docs/en/{version}/')
    map.connect('doc_home', '/docs/{language}/{version}/', controller='docs', action='view', url='index', language='en',
                version=globs.doc_version)
    map.connect('doc_view', '/docs/{language}/{version}/{url:.*}', controller='docs', action='view', language='en',
                version=globs.doc_version)

    map.connect('wiki', 'http://wiki.pylonshq.com/', _static=True)
    map.connect('download', '/download/{version}', controller='download',
                action='index', version=globs.doc_version)
    map.connect('/download', controller='download', action='index', version=version)
    map.connect('cdocs', '%s/display/pylonsdocs/{page}' % wiki, _static=True)

    # Accounts
    map.connect('account_login', '/accounts/login', controller='accounts', action='login')
    map.connect('account_register', '/accounts/register', controller='accounts', action='register')
    map.connect('account_logout', '/accounts/logout', controller='accounts', action='logout')
    map.connect('verify_email', '/accounts/verify_email/{token}', controller='accounts', action='verify_email')
    map.connect('forgot_password', '/accounts/forgot_password', controller='accounts', action='forgot_password')
    map.connect('reset_password', '/accounts/reset_password/{token}', controller='accounts', action='change_password')

    # OpenID URL's
    map.connect('openid_associate', '/accounts/openid/associate', controller='accounts', action='openid_associate')
    map.connect('openid_register', '/accounts/openid/register', controller='accounts', action='openid_register')
    map.connect('openid_login', '/accounts/openid/login', controller='consumer', action='login')
    map.connect('openid_process', '/accounts/openid/process', controller='consumer', action='process')
    map.connect('openid_create', '/accounts/openid/create', controller='consumer', action='create')

    # Snippets
    map.connect('snippet_author', '/snippets/by_author/{id}', controller='snippets', action='by_author')
    map.connect('snippet_tag', '/snippets/by_tag/{tag}', controller='snippets', action='by_tag')
    map.connect('snippet_tagcloud', '/snippets/tagcloud', controller='snippets', action='tagcloud')
    map.resource('snippet', 'snippets', collection={'preview':'POST'})

    # Pastebin
    map.connect('pasties_tagcloud', '/pasties/tagcloud', controller='pasties', action='tagcloud')
    map.connect('formatted_pasties_tag', '/pasties/by_tag/{tag}.{format}', controller='pasties', action='index')
    map.connect('pasties_tag', '/pasties/by_tag/{tag}', controller='pasties', action='index')
    map.connect('pasties_author', '/pasties/by_author/{author}', controller='pasties', action='by_author')
    map.resource('paste', 'pasties')

    # Resources
    map.resource('traceback', 'tracebacks', member={'reown':'GET'})

    # Comments
    map.connect('preview_comment', '/comment/preview', controller='comments', action='preview', conditions=dict(method='POST'))
    map.connect('post_comment', '/comment/{doc_id}', controller='comments', action='create', conditions=dict(method='POST'))
    map.connect('delete_comment', '/comment/{id}', controller='comments', action='delete', conditions=dict(method='DELETE'))
    map.resource('comment', 'comments')

    # Trac
    map.connect('/project/*path_info', controller='tracs', action='run_app')
    map.connect('trac_link', '/project/pylonshq/*url')

    # External Links
    map.connect('sqlalchemy', 'http://sqlalchemy.org/', _static=True)
    map.connect('sqlobject', 'http://www.sqlobject.org/', _static=True)
    map.connect('python-couchdb', 'http://code.google.com/p/couchdb-python/', _static=True)
    map.connect('formalchemy', 'http://code.google.com/p/formalchemy/', _static=True)
    map.connect('mako', 'http://www.makotemplates.org/', _static=True)
    map.connect('genshi', 'http://genshi.edgewall.org/', _static=True)
    map.connect('jinja2', 'http://jinja.pocoo.org/', _static=True)
    map.connect('pylons_book', 'http://pylonsbook.com/', _static=True)

    map.connect('/{controller}/{action}')
    map.connect('/{controller}/{action}/{id}')

    return map
<commit_msg>Fix download link<commit_after>"""Routes configuration

The more specific and detailed routes should be defined first so they
may take precedent over the more generic routes. For more information
refer to the routes manual at http://routes.groovie.org/docs/
"""
from pylons import config
from routes import Mapper

wiki = 'http://wiki.pylonshq.com'

def article_expand(kwargs):
    if 'article' not in kwargs:
        return kwargs
    article = kwargs.pop('article')
    kwargs['year'] = article.published.year
    kwargs['month'] = article.published.month
    kwargs['slug'] = article.slug
    return kwargs

def make_map(globs=None):
    """Create, configure and return the routes Mapper"""
    version = config['pylons.app_globals'].current_version
    static_host = config['cdn.uri']

    map = Mapper(directory=config['pylons.paths']['controllers'],
                 always_scan=config['debug'])
    map.minimization = False

    # The ErrorController route (handles 404/500 error pages); it should
    # likely stay at the top, ensuring it can always be resolved
    map.connect('/error/{action}', controller='error')
    map.connect('/error/{action}/{id}', controller='error')

    # Home url's
    map.connect('home', '/', controller='home', action='index')
    map.connect('features', '/features', controller='home', action='features')
    map.connect('history', '/history', controller='home', action='history')

    # Blog url's
    map.connect('article_archives', '/articles/archives/{year:\d+}/{month:\d+}/{slug}',
                controller='articles', action='archives', _filter=article_expand)
    map.resource('article', 'articles')

    # Code url's
    map.connect('buildbot', '/buildbot/{action}', controller='buildbot')

    # Doc url's
    map.connect('doc_upload', '/docs/upload', controller='docs', action='upload')
    map.connect('doc_upload_image', '/docs/upload_image', controller='docs', action='upload_image')
    map.connect('doc_delete', '/docs/delete_revision/{project}/{version}', controller='docs', action='delete_revision')
    map.redirect('/docs/{version}', '/docs/en/{version}/')
    map.connect('doc_home', '/docs/{language}/{version}/', controller='docs', action='view', url='index', language='en',
                version=globs.doc_version)
    map.connect('doc_view', '/docs/{language}/{version}/{url:.*}', controller='docs', action='view', language='en',
                version=globs.doc_version)

    map.connect('wiki', 'http://wiki.pylonshq.com/', _static=True)
    map.connect('download', '/download/{version}', controller='download',
                action='index', version=globs.doc_version)
    map.redirect('/download', '/download/{version}', version=globs.versions[-1])
    map.connect('cdocs', '%s/display/pylonsdocs/{page}' % wiki, _static=True)

    # Accounts
    map.connect('account_login', '/accounts/login', controller='accounts', action='login')
    map.connect('account_register', '/accounts/register', controller='accounts', action='register')
    map.connect('account_logout', '/accounts/logout', controller='accounts', action='logout')
    map.connect('verify_email', '/accounts/verify_email/{token}', controller='accounts', action='verify_email')
    map.connect('forgot_password', '/accounts/forgot_password', controller='accounts', action='forgot_password')
    map.connect('reset_password', '/accounts/reset_password/{token}', controller='accounts', action='change_password')

    # OpenID URL's
    map.connect('openid_associate', '/accounts/openid/associate', controller='accounts', action='openid_associate')
    map.connect('openid_register', '/accounts/openid/register', controller='accounts', action='openid_register')
    map.connect('openid_login', '/accounts/openid/login', controller='consumer', action='login')
    map.connect('openid_process', '/accounts/openid/process', controller='consumer', action='process')
    map.connect('openid_create', '/accounts/openid/create', controller='consumer', action='create')

    # Snippets
    map.connect('snippet_author', '/snippets/by_author/{id}', controller='snippets', action='by_author')
    map.connect('snippet_tag', '/snippets/by_tag/{tag}', controller='snippets', action='by_tag')
    map.connect('snippet_tagcloud', '/snippets/tagcloud', controller='snippets', action='tagcloud')
    map.resource('snippet', 'snippets', collection={'preview':'POST'})

    # Pastebin
    map.connect('pasties_tagcloud', '/pasties/tagcloud', controller='pasties', action='tagcloud')
    map.connect('formatted_pasties_tag', '/pasties/by_tag/{tag}.{format}', controller='pasties', action='index')
    map.connect('pasties_tag', '/pasties/by_tag/{tag}', controller='pasties', action='index')
    map.connect('pasties_author', '/pasties/by_author/{author}', controller='pasties', action='by_author')
    map.resource('paste', 'pasties')

    # Resources
    map.resource('traceback', 'tracebacks', member={'reown':'GET'})

    # Comments
    map.connect('preview_comment', '/comment/preview', controller='comments', action='preview', conditions=dict(method='POST'))
    map.connect('post_comment', '/comment/{doc_id}', controller='comments', action='create', conditions=dict(method='POST'))
    map.connect('delete_comment', '/comment/{id}', controller='comments', action='delete', conditions=dict(method='DELETE'))
    map.resource('comment', 'comments')

    # Trac
    map.connect('/project/*path_info', controller='tracs', action='run_app')
    map.connect('trac_link', '/project/pylonshq/*url')

    # External Links
    map.connect('sqlalchemy', 'http://sqlalchemy.org/', _static=True)
    map.connect('sqlobject', 'http://www.sqlobject.org/', _static=True)
    map.connect('python-couchdb', 'http://code.google.com/p/couchdb-python/', _static=True)
    map.connect('formalchemy', 'http://code.google.com/p/formalchemy/', _static=True)
    map.connect('mako', 'http://www.makotemplates.org/', _static=True)
    map.connect('genshi', 'http://genshi.edgewall.org/', _static=True)
    map.connect('jinja2', 'http://jinja.pocoo.org/', _static=True)
    map.connect('pylons_book', 'http://pylonsbook.com/', _static=True)

    map.connect('/{controller}/{action}')
    map.connect('/{controller}/{action}/{id}')

    return map
<|endoftext|>


<commit_before>import fiona
import geohash
import os
import rtree
import six
import ujson as json

from collections import OrderedDict, defaultdict
from leveldb import LevelDB
from lru import LRU
from shapely.geometry import Point, Polygon, MultiPolygon
from shapely.prepared import prep
from shapely.geometry.geo import mapping

from geodata.polygons.area import polygon_bounding_box_area

DEFAULT_POLYS_FILENAME = 'polygons.geojson'
DEFAULT_PROPS_FILENAME = 'properties.json'

class PolygonIndex(object):
    include_only_properties = None
    simplify_tolerance = 0.0001
    preserve_topology = True
    persistent_polygons = False
    cache_size = 0

    INDEX_FILENAME = None
    POLYGONS_DB_DIR = 'polygons'

    def __init__(self, index=None, polygons=None, polygons_db=None, save_dir=None,
                 index_filename=None,
                 polygons_db_path=None,
                 include_only_properties=None):
        if save_dir:
            self.save_dir = save_dir
        else:
            self.save_dir = None

        if not index_filename:
            index_filename = self.INDEX_FILENAME

        self.index_path = os.path.join(save_dir or '.', index_filename)

        if not index:
            self.create_index(overwrite=True)
        else:
            self.index = index

        if include_only_properties and hasattr(include_only_properties, '__contains__'):
            self.include_only_properties = include_only_properties

        if not polygons and not self.persistent_polygons:
            self.polygons = {}
        elif polygons and not self.persistent_polygons:
            self.polygons = polygons
        elif self.persistent_polygons and self.cache_size > 0:
            self.polygons = LRU(self.cache_size)
            if polygons:
                for key, value in six.iteritems(polygons):
                    self.polygons[key] = value

            self.cache_hits = 0
            self.cache_misses = 0

            self.get_polygon = self.get_polygon_cached

        if not polygons_db_path:
            polygons_db_path = os.path.join(save_dir or '.', self.POLYGONS_DB_DIR)

        if not polygons_db:
            self.polygons_db = LevelDB(polygons_db_path)
        else:
            self.polygons_db = polygons_db

        self.setup()

        self.i = 0

    def create_index(self, overwrite=False):
        raise NotImplementedError('Children must implement')

    def index_polygon(self, polygon):
        raise NotImplementedError('Children must implement')

    def setup(self):
        pass

    def simplify_polygon(self, poly, simplify_tolerance=None, preserve_topology=None):
        if simplify_tolerance is None:
            simplify_tolerance = self.simplify_tolerance
        if preserve_topology is None:
            preserve_topology = self.preserve_topology
        return poly.simplify(simplify_tolerance, preserve_topology=preserve_topology)

    def index_polygon_properties(self, properties):
        pass

    def polygon_geojson(self, poly, properties):
        return {
            'type': 'Feature',
            'geometry': mapping(poly),
        }

    def add_polygon(self, poly, properties, cache=False, include_only_properties=None):
        if include_only_properties is not None:
            properties = {k: v for k, v in properties.iteritems() if k in include_only_properties}

        if not self.persistent_polygons or cache:
            self.polygons[self.i] = prep(poly)

        if self.persistent_polygons:
            self.polygons_db.Put(self.polygon_key(self.i), json.dumps(self.polygon_geojson(poly, properties)))

        self.polygons_db.Put(self.properties_key(self.i), json.dumps(properties))
        self.index_polygon_properties(properties)
        self.i += 1

    @classmethod
    def create_from_shapefiles(cls, inputs, output_dir,
                               index_filename=None,
                               include_only_properties=None):
        index = cls(save_dir=output_dir, index_filename=index_filename or cls.INDEX_FILENAME)
        for input_file in inputs:
            if include_only_properties is not None:
                include_props = include_only_properties.get(input_file, cls.include_only_properties)
            else:
                include_props = cls.include_only_properties

            f = fiona.open(input_file)

            index.add_geojson_like_file(f)

        return index

    @classmethod
    def fix_polygon(cls, poly):
        '''
        Coerce to valid polygon
        '''
        if not poly.is_valid:
            poly = poly.buffer(0)
            if not poly.is_valid:
                return None
        return poly

    @classmethod
    def to_polygon(cls, coords):
        '''
        Create shapely polygon from list of coordinate tuples if valid
        '''
        if not coords or len(coords) < 3:
            return None
        poly = Polygon(coords)
        return poly

    def add_geojson_like_record(self, rec, include_only_properties=None):
        if not rec or not rec.get('geometry') or 'type' not in rec['geometry']:
            return
        poly_type = rec['geometry']['type']
        if poly_type == 'Polygon':
            coords = rec['geometry']['coordinates'][0]
            poly = self.to_polygon(coords)
            if poly is None or not poly.bounds or len(poly.bounds) != 4:
                return
            self.index_polygon(poly)
            self.add_polygon(poly, rec['properties'], include_only_properties=include_only_properties)
        elif poly_type == 'MultiPolygon':
            polys = []
            poly_coords = rec['geometry']['coordinates']
            for coords in poly_coords:
                poly = self.to_polygon(coords[0])
                if poly is None or not poly.bounds or len(poly.bounds) != 4:
                    continue
                polys.append(poly)
                self.index_polygon(poly)

            self.add_polygon(MultiPolygon(polys), rec['properties'], include_only_properties=include_only_properties)
        else:
            return

    def add_geojson_like_file(self, f, include_only_properties=None):
        '''
        Add either GeoJSON or a shapefile record to the index
        '''

        for rec in f:
            self.add_geojson_like_record(rec, include_only_properties=include_only_properties)

    @classmethod
    def create_from_geojson_files(cls, inputs, output_dir,
                                  index_filename=None,
                                  polys_filename=DEFAULT_POLYS_FILENAME,
                                  include_only_properties=None):
        index = cls(save_dir=output_dir, index_filename=index_filename or cls.INDEX_FILENAME)
        for input_file in inputs:
            if include_only_properties is not None:
                include_props = include_only_properties.get(input_file, cls.include_only_properties)
            else:
                include_props = cls.include_only_properties

            f = json.load(open(input_file))

            index.add_geojson_like_file(f['features'], include_only_properties=include_props)

        return index

    def compact_polygons_db(self):
        self.polygons_db.CompactRange('\x00', '\xff')

    def save(self):
        self.save_index()
        self.save_properties(os.path.join(self.save_dir, DEFAULT_PROPS_FILENAME))
        if not self.persistent_polygons:
            self.save_polygons(os.path.join(self.save_dir, DEFAULT_POLYS_FILENAME))
        self.compact_polygons_db()
        self.save_polygon_properties(self.save_dir)

    def load_properties(self, filename):
        properties = json.load(open(filename))
        self.i = int(properties.get('num_polygons', self.i))

    def save_properties(self, out_filename):
        out = open(out_filename, 'w')
        json.dump({'num_polygons': str(self.i)})

    def save_polygons(self, out_filename):
        out = open(out_filename, 'w')
        for i in xrange(self.i):
            poly = self.polygons[i]
            feature = {
                'type': 'Feature',
                'geometry': mapping(poly.context),
            }
            out.write(json.dumps(feature) + u'\n')

    def save_index(self):
        raise NotImplementedError('Children must implement')

    def load_polygon_properties(self, d):
        pass

    def save_polygon_properties(self, d):
        pass

    @classmethod
    def polygon_from_geojson(cls, feature):
        poly_type = feature['geometry']['type']
        if poly_type == 'Polygon':
            poly = Polygon(feature['geometry']['coordinates'][0])
            return poly
        elif poly_type == 'MultiPolygon':
            polys = []
            for coords in feature['geometry']['coordinates']:
                poly = Polygon(coords[0])
                polys.append(poly)

            return MultiPolygon(polys)

    @classmethod
    def load_polygons(cls, filename):
        f = open(filename)
        polygons = {}
        cls.i = 0
        for line in f:
            feature = json.loads(line.rstrip())
            polygons[cls.i] = prep(cls.polygon_from_geojson(feature))
            cls.i += 1
        return polygons

    @classmethod
    def load_index(cls, d, index_name=None):
        raise NotImplementedError('Children must implement')

    @classmethod
    def load(cls, d, index_name=None, polys_filename=DEFAULT_POLYS_FILENAME,
             properties_filename=DEFAULT_PROPS_FILENAME,
             polys_db_dir=POLYGONS_DB_DIR):
        index = cls.load_index(d, index_name=index_name or cls.INDEX_FILENAME)
        if not cls.persistent_polygons:
            polys = cls.load_polygons(os.path.join(d, polys_filename))
        else:
            polys = None
        polygons_db = LevelDB(os.path.join(d, polys_db_dir))
        polygon_index = cls(index=index, polygons=polys, polygons_db=polygons_db, save_dir=d)
        polygon_index.load_properties(os.path.join(d, properties_filename))
        polygon_index.load_polygon_properties(d)
        return polygon_index

    def get_candidate_polygons(self, lat, lon):
        raise NotImplementedError('Children must implement')

    def get_properties(self, i):
        return json.loads(self.polygons_db.Get(self.properties_key(i)))

    def get_polygon(self, i):
        return self.polygons[i]

    def get_polygon_cached(self, i):
        poly = self.polygons.get(i, None)
        if poly is None:
            data = json.loads(self.polygons_db.Get(self.polygon_key(i)))
            poly = prep(self.polygon_from_geojson(data))
            self.polygons[i] = poly
            self.cache_misses += 1
        else:
            self.cache_hits += 1
        return poly

    def __iter__(self):
        for i in xrange(self.i):
            yield self.get_properties(i), self.get_polygon(i)

    def __len__(self):
        return self.i

    def polygons_contain(self, candidates, point, return_all=False):
        containing = None
        if return_all:
            containing = []
        for i in candidates:
            poly = self.get_polygon(i)
            contains = poly.contains(point)
            if contains:
                properties = self.get_properties(i)
                if not return_all:
                    return properties
                else:
                    containing.append(properties)
        return containing

    def polygon_key(self, i):
        return 'poly:{}'.format(i)

    def properties_key(self, i):
        return 'props:{}'.format(i)

    def point_in_poly(self, lat, lon, return_all=False):
        candidates = self.get_candidate_polygons(lat, lon)
        point = Point(lon, lat)
        return self.polygons_contain(candidates, point, return_all=return_all)

class RTreePolygonIndex(PolygonIndex):
    INDEX_FILENAME = 'rtree'

    def create_index(self, overwrite=False):
        self.index = rtree.index.Index(self.index_path, overwrite=overwrite)

    def index_polygon(self, polygon):
        self.index.insert(self.i, polygon.bounds)

    def get_candidate_polygons(self, lat, lon):
        return OrderedDict.fromkeys(self.index.intersection((lon, lat, lon, lat))).keys()

    def save_index(self):
        # need to close index before loading it
        self.index.close()

    @classmethod
    def load_index(cls, d, index_name=None):
        return rtree.index.Index(os.path.join(d, index_name or cls.INDEX_FILENAME))

class GeohashPolygonIndex(PolygonIndex):
    # Reference: https://www.elastic.co/guide/en/elasticsearch/guide/current/geohashes.html
    GEOHASH_PRECISIONS = [
        (7, 152.8 ** 2),
        (6, 1200.0 * 610.0),
        (5, 4900.0 * 4900.0),
        (4, 39000.0 * 19500.0),
        (3, 156000.0 * 156000.0),
        (2, 1251000.0 * 625000.0),
    ]

    INDEX_FILENAME = 'index.json'

    def create_index(self, overwrite=False):
        self.index = defaultdict(list)

    def index_point(self, lat, lon, geohash_level):
        code = geohash.encode(lat, lon)[:geohash_level]

        for key in [code] + geohash.neighbors(code):
            self.index[key].append(self.i)

    def index_polygon(self, polygon):
        poly_area = polygon_bounding_box_area(polygon)
        for geohash_level, area in self.GEOHASH_PRECISIONS:
            if area * 9.0 >= poly_area:
                break

        bbox = polygon.bounds
        lat, lon = (bbox[1] + bbox[3]) / 2.0, (bbox[0] + bbox[2]) / 2.0
        self.index_point(lat, lon, geohash_level)

    def get_candidate_polygons(self, lat, lon, return_all=False):
        candidates = []
        code = geohash.encode(lat, lon)
        for level, area in self.GEOHASH_PRECISIONS:
            indices = self.index.get(code[:level])
            if not indices:
                continue
            candidates.extend(indices)
            if not return_all:
                break
        return candidates

    def save_index(self):
        if not self.index_path:
            self.index_path = os.path.join(self.save_dir or '.', self.INDEX_FILENAME)
        json.dump(self.index, open(self.index_path, 'w'))

    @classmethod
    def load_index(cls, d, index_name=None):
        return json.load(open(os.path.join(d, index_name or cls.INDEX_FILENAME)))
<commit_msg>[fix] polygons<commit_after>import fiona
import geohash
import os
import rtree
import six
import ujson as json

from collections import OrderedDict, defaultdict
from leveldb import LevelDB
from lru import LRU
from shapely.geometry import Point, Polygon, MultiPolygon
from shapely.prepared import prep
from shapely.geometry.geo import mapping

from geodata.polygons.area import polygon_bounding_box_area

DEFAULT_POLYS_FILENAME = 'polygons.geojson'
DEFAULT_PROPS_FILENAME = 'properties.json'

class PolygonIndex(object):
    include_only_properties = None
    simplify_tolerance = 0.0001
    preserve_topology = True
    persistent_polygons = False
    cache_size = 0

    INDEX_FILENAME = None
    POLYGONS_DB_DIR = 'polygons'

    def __init__(self, index=None, polygons=None, polygons_db=None, save_dir=None,
                 index_filename=None,
                 polygons_db_path=None,
                 include_only_properties=None):
        if save_dir:
            self.save_dir = save_dir
        else:
            self.save_dir = None

        if not index_filename:
            index_filename = self.INDEX_FILENAME

        self.index_path = os.path.join(save_dir or '.', index_filename)

        if not index:
            self.create_index(overwrite=True)
        else:
            self.index = index

        if include_only_properties and hasattr(include_only_properties, '__contains__'):
            self.include_only_properties = include_only_properties

        if not polygons and not self.persistent_polygons:
            self.polygons = {}
        elif polygons and not self.persistent_polygons:
            self.polygons = polygons
        elif self.persistent_polygons and self.cache_size > 0:
            self.polygons = LRU(self.cache_size)
            if polygons:
                for key, value in six.iteritems(polygons):
                    self.polygons[key] = value

            self.cache_hits = 0
            self.cache_misses = 0

            self.get_polygon = self.get_polygon_cached

        if not polygons_db_path:
            polygons_db_path = os.path.join(save_dir or '.', self.POLYGONS_DB_DIR)

        if not polygons_db:
            self.polygons_db = LevelDB(polygons_db_path)
        else:
            self.polygons_db = polygons_db

        self.setup()

        self.i = 0

    def create_index(self, overwrite=False):
        raise NotImplementedError('Children must implement')

    def index_polygon(self, polygon):
        raise NotImplementedError('Children must implement')

    def setup(self):
        pass

    def simplify_polygon(self, poly, simplify_tolerance=None, preserve_topology=None):
        if simplify_tolerance is None:
            simplify_tolerance = self.simplify_tolerance
        if preserve_topology is None:
            preserve_topology = self.preserve_topology
        return poly.simplify(simplify_tolerance, preserve_topology=preserve_topology)

    def index_polygon_properties(self, properties):
        pass

    def polygon_geojson(self, poly, properties):
        return {
            'type': 'Feature',
            'geometry': mapping(poly),
        }

    def add_polygon(self, poly, properties, cache=False, include_only_properties=None):
        if include_only_properties is not None:
            properties = {k: v for k, v in properties.iteritems() if k in include_only_properties}

        if not self.persistent_polygons or cache:
            self.polygons[self.i] = prep(poly)

        if self.persistent_polygons:
            self.polygons_db.Put(self.polygon_key(self.i), json.dumps(self.polygon_geojson(poly, properties)))

        self.polygons_db.Put(self.properties_key(self.i), json.dumps(properties))
        self.index_polygon_properties(properties)
        self.i += 1

    @classmethod
    def create_from_shapefiles(cls, inputs, output_dir,
                               index_filename=None,
                               include_only_properties=None):
        index = cls(save_dir=output_dir, index_filename=index_filename or cls.INDEX_FILENAME)
        for input_file in inputs:
            if include_only_properties is not None:
                include_props = include_only_properties.get(input_file, cls.include_only_properties)
            else:
                include_props = cls.include_only_properties

            f = fiona.open(input_file)

            index.add_geojson_like_file(f)

        return index

    @classmethod
    def fix_polygon(cls, poly):
        '''
        Coerce to valid polygon
        '''
        if not poly.is_valid:
            poly = poly.buffer(0)
            if not poly.is_valid:
                return None
        return poly

    @classmethod
    def to_polygon(cls, coords):
        '''
        Create shapely polygon from list of coordinate tuples if valid
        '''
        if not coords or len(coords) < 3:
            return None
        poly = Polygon(coords)
        return poly

    def add_geojson_like_record(self, rec, include_only_properties=None):
        if not rec or not rec.get('geometry') or 'type' not in rec['geometry']:
            return
        poly_type = rec['geometry']['type']
        if poly_type == 'Polygon':
            coords = rec['geometry']['coordinates'][0]
            poly = self.to_polygon(coords)
            if poly is None or not poly.bounds or len(poly.bounds) != 4:
                return
            self.index_polygon(poly)
            self.add_polygon(poly, rec['properties'], include_only_properties=include_only_properties)
        elif poly_type == 'MultiPolygon':
            polys = []
            poly_coords = rec['geometry']['coordinates']
            for coords in poly_coords:
                poly = self.to_polygon(coords[0])
                if poly is None or not poly.bounds or len(poly.bounds) != 4:
                    continue
                polys.append(poly)
                self.index_polygon(poly)

            self.add_polygon(MultiPolygon(polys), rec['properties'], include_only_properties=include_only_properties)
        else:
            return

    def add_geojson_like_file(self, f, include_only_properties=None):
        '''
        Add either GeoJSON or a shapefile record to the index
        '''

        for rec in f:
            self.add_geojson_like_record(rec, include_only_properties=include_only_properties)

    @classmethod
    def create_from_geojson_files(cls, inputs, output_dir,
                                  index_filename=None,
                                  polys_filename=DEFAULT_POLYS_FILENAME,
                                  include_only_properties=None):
        index = cls(save_dir=output_dir, index_filename=index_filename or cls.INDEX_FILENAME)
        for input_file in inputs:
            if include_only_properties is not None:
                include_props = include_only_properties.get(input_file, cls.include_only_properties)
            else:
                include_props = cls.include_only_properties

            f = json.load(open(input_file))

            index.add_geojson_like_file(f['features'], include_only_properties=include_props)

        return index

    def compact_polygons_db(self):
        self.polygons_db.CompactRange('\x00', '\xff')

    def save(self):
        self.save_index()
        self.save_properties(os.path.join(self.save_dir, DEFAULT_PROPS_FILENAME))
        if not self.persistent_polygons:
            self.save_polygons(os.path.join(self.save_dir, DEFAULT_POLYS_FILENAME))
        self.compact_polygons_db()
        self.save_polygon_properties(self.save_dir)

    def load_properties(self, filename):
        properties = json.load(open(filename))
        self.i = int(properties.get('num_polygons', self.i))

    def save_properties(self, out_filename):
        out = open(out_filename, 'w')
        json.dump({'num_polygons': str(self.i)}, out)

    def save_polygons(self, out_filename):
        out = open(out_filename, 'w')
        for i in xrange(self.i):
            poly = self.polygons[i]
            feature = {
                'type': 'Feature',
                'geometry': mapping(poly.context),
            }
            out.write(json.dumps(feature) + u'\n')

    def save_index(self):
        raise NotImplementedError('Children must implement')

    def load_polygon_properties(self, d):
        pass

    def save_polygon_properties(self, d):
        pass

    @classmethod
    def polygon_from_geojson(cls, feature):
        poly_type = feature['geometry']['type']
        if poly_type == 'Polygon':
            poly = Polygon(feature['geometry']['coordinates'][0])
            return poly
        elif poly_type == 'MultiPolygon':
            polys = []
            for coords in feature['geometry']['coordinates']:
                poly = Polygon(coords[0])
                polys.append(poly)

            return MultiPolygon(polys)

    @classmethod
    def load_polygons(cls, filename):
        f = open(filename)
        polygons = {}
        cls.i = 0
        for line in f:
            feature = json.loads(line.rstrip())
            polygons[cls.i] = prep(cls.polygon_from_geojson(feature))
            cls.i += 1
        return polygons

    @classmethod
    def load_index(cls, d, index_name=None):
        raise NotImplementedError('Children must implement')

    @classmethod
    def load(cls, d, index_name=None, polys_filename=DEFAULT_POLYS_FILENAME,
             properties_filename=DEFAULT_PROPS_FILENAME,
             polys_db_dir=POLYGONS_DB_DIR):
        index = cls.load_index(d, index_name=index_name or cls.INDEX_FILENAME)
        if not cls.persistent_polygons:
            polys = cls.load_polygons(os.path.join(d, polys_filename))
        else:
            polys = None
        polygons_db = LevelDB(os.path.join(d, polys_db_dir))
        polygon_index = cls(index=index, polygons=polys, polygons_db=polygons_db, save_dir=d)
        polygon_index.load_properties(os.path.join(d, properties_filename))
        polygon_index.load_polygon_properties(d)
        return polygon_index

    def get_candidate_polygons(self, lat, lon):
        raise NotImplementedError('Children must implement')

    def get_properties(self, i):
        return json.loads(self.polygons_db.Get(self.properties_key(i)))

    def get_polygon(self, i):
        return self.polygons[i]

    def get_polygon_cached(self, i):
        poly = self.polygons.get(i, None)
        if poly is None:
            data = json.loads(self.polygons_db.Get(self.polygon_key(i)))
            poly = prep(self.polygon_from_geojson(data))
            self.polygons[i] = poly
            self.cache_misses += 1
        else:
            self.cache_hits += 1
        return poly

    def __iter__(self):
        for i in xrange(self.i):
            yield self.get_properties(i), self.get_polygon(i)

    def __len__(self):
        return self.i

    def polygons_contain(self, candidates, point, return_all=False):
        containing = None
        if return_all:
            containing = []
        for i in candidates:
            poly = self.get_polygon(i)
            contains = poly.contains(point)
            if contains:
                properties = self.get_properties(i)
                if not return_all:
                    return properties
                else:
                    containing.append(properties)
        return containing

    def polygon_key(self, i):
        return 'poly:{}'.format(i)

    def properties_key(self, i):
        return 'props:{}'.format(i)

    def point_in_poly(self, lat, lon, return_all=False):
        candidates = self.get_candidate_polygons(lat, lon)
        point = Point(lon, lat)
        return self.polygons_contain(candidates, point, return_all=return_all)

class RTreePolygonIndex(PolygonIndex):
    INDEX_FILENAME = 'rtree'

    def create_index(self, overwrite=False):
        self.index = rtree.index.Index(self.index_path, overwrite=overwrite)

    def index_polygon(self, polygon):
        self.index.insert(self.i, polygon.bounds)

    def get_candidate_polygons(self, lat, lon):
        return OrderedDict.fromkeys(self.index.intersection((lon, lat, lon, lat))).keys()

    def save_index(self):
        # need to close index before loading it
        self.index.close()

    @classmethod
    def load_index(cls, d, index_name=None):
        return rtree.index.Index(os.path.join(d, index_name or cls.INDEX_FILENAME))

class GeohashPolygonIndex(PolygonIndex):
    # Reference: https://www.elastic.co/guide/en/elasticsearch/guide/current/geohashes.html
    GEOHASH_PRECISIONS = [
        (7, 152.8 ** 2),
        (6, 1200.0 * 610.0),
        (5, 4900.0 * 4900.0),
        (4, 39000.0 * 19500.0),
        (3, 156000.0 * 156000.0),
        (2, 1251000.0 * 625000.0),
    ]

    INDEX_FILENAME = 'index.json'

    def create_index(self, overwrite=False):
        self.index = defaultdict(list)

    def index_point(self, lat, lon, geohash_level):
        code = geohash.encode(lat, lon)[:geohash_level]

        for key in [code] + geohash.neighbors(code):
            self.index[key].append(self.i)

    def index_polygon(self, polygon):
        poly_area = polygon_bounding_box_area(polygon)
        for geohash_level, area in self.GEOHASH_PRECISIONS:
            if area * 9.0 >= poly_area:
                break

        bbox = polygon.bounds
        lat, lon = (bbox[1] + bbox[3]) / 2.0, (bbox[0] + bbox[2]) / 2.0
        self.index_point(lat, lon, geohash_level)

    def get_candidate_polygons(self, lat, lon, return_all=False):
        candidates = []
        code = geohash.encode(lat, lon)
        for level, area in self.GEOHASH_PRECISIONS:
            indices = self.index.get(code[:level])
            if not indices:
                continue
            candidates.extend(indices)
            if not return_all:
                break
        return candidates

    def save_index(self):
        if not self.index_path:
            self.index_path = os.path.join(self.save_dir or '.', self.INDEX_FILENAME)
        json.dump(self.index, open(self.index_path, 'w'))

    @classmethod
    def load_index(cls, d, index_name=None):
        return json.load(open(os.path.join(d, index_name or cls.INDEX_FILENAME)))
<|endoftext|>


<commit_before>from .input_devices import Button
from .output_devices import LED
from .devices import GPIODeviceError

class TrafficLights(object):
    def __init__(self, red=None, amber=None, green=None):
        if not all([red, amber, green]):
            raise GPIODeviceError('Red, Amber and Green pins must be provided')

        self.red = LED(red)
        self.amber = LED(amber)
        self.green = LED(green)
        self.lights = [self.red, self.amber, self.green]

    def lights_on(self):
        for led in self.lights:
            led.on()

    def lights_off(self):
        for led in self.lights:
            led.off()

class PiTraffic(TrafficLights):
    def __init__(self):
        self.red = LED(9)
        self.amber = LED(10)
        self.green = LED(11)
        self.lights = [self.red, self.amber, self.green]

class FishDish(TrafficLights):
    def __init__(self):
        red, amber, green = (9, 22, 4)
        super(FishDish, self).__init__(red, amber, green)
        self.buzzer = Buzzer(8)
        self.button = Button(7)
        self.all = self.lights + [self.buzzer]

    def on(self):
        for led in self.all:
            led.on()

    def off(self):
        for led in self.all:
            led.off()

class PiLiter(object):
    def __init__(self):
        leds = [4, 17, 27, 18, 22, 23, 24, 25]
        self.leds = [LED(led) for led in leds]

    def on(self):
        for led in self.leds:
            led.on()

    def off(self):
        for led in self.leds:
            led.off()
<commit_msg>Prefix lights and all properties and convert lists to tuples<commit_after>from .input_devices import Button
from .output_devices import LED
from .devices import GPIODeviceError

class TrafficLights(object):
    def __init__(self, red=None, amber=None, green=None):
        if not all([red, amber, green]):
            raise GPIODeviceError('Red, Amber and Green pins must be provided')

        self.red = LED(red)
        self.amber = LED(amber)
        self.green = LED(green)
        self._lights = (self.red, self.amber, self.green)

    def on(self):
        for led in self._lights:
            led.on()

    def off(self):
        for led in self._lights:
            led.off()

class PiTraffic(TrafficLights):
    def __init__(self):
        self.red = LED(9)
        self.amber = LED(10)
        self.green = LED(11)
        self._lights = (self.red, self.amber, self.green)

class FishDish(TrafficLights):
    def __init__(self):
        red, amber, green = (9, 22, 4)
        super(FishDish, self).__init__(red, amber, green)
        self.buzzer = Buzzer(8)
        self.button = Button(7)
        self._all = tuple(list(self._lights) + [self.buzzer])

    def on(self):
        for thing in self._all:
            thing.on()

    def off(self):
        for thing in self._all:
            thing.off()

    def lights_on(self):
        super.on()

    def lights_off(self):
        super.off()

class PiLiter(object):
    def __init__(self):
        leds = [4, 17, 27, 18, 22, 23, 24, 25]
        self.leds = tuple([LED(led) for led in leds])

    def on(self):
        for led in self.leds:
            led.on()

    def off(self):
        for led in self.leds:
            led.off()
<|endoftext|>


<commit_before>from falafel.core import MapperOutput
from falafel.core.plugins import mapper

import re

@mapper('smartctl')
class SMARTctl(MapperOutput):
    """
        Mapper for output of smartctl -a for each drive in system.
        This returns a dict with keys:
         * 'device' - the name of the device after /dev/ - e.g. sda
         * 'info' - the -i info (vendor, product, etc)
         * 'health' - overall health assessment (-H)
         * 'values' - the SMART values (-c) - SMART config on drive firmware
         * 'attributes' - the SMART attributes (-A) - run time data
        This function gets called once per detected file, so we don't have to
        deal with file changes within this process.
    """

    def __init__(self, data, path):
        filename_re = re.compile(r'smartctl_-a_\.dev\.(?P<device>\w+)$')
        match = filename_re.search(path)
        if match:
            data['device'] = match.group('device')
        # Else warn?  fail?

        super(SMARTctl, self).__init__(data, path)

    @staticmethod
    def parse_content(content):
        drive_info = {
            'info'          : {},
            'health'        : 'not parsed',
            'values'        : {},
            'attributes'    : {},
            'full_line'     : '', # hack for persistent line storage
        }

        info_line_re =  re.compile(r'(?P<key>\w+(?:\s\w+)*):\s+' +
         r'(?P<value>\S.*?)\s*$')
        value_line_re = re.compile(r'(?P<key>\w[A-Za-z _.-]+):\s+' +
         r'\(\s*(?P<value>\S.*?)\)')
        attr_line_re =  re.compile(r'^\s*(?P<id>\d+)\s(?P<name>\w+)\s+' +
         r'(?P<flag>0x[0-9a-fA-F]{4})\s+(?P<value>\d{3})\s+' +
         r'(?P<worst>\d{3})\s+(?P<threshold>\d{3})\s+' +
         r'(?P<type>[A-Za-z_-]+)\s+(?P<updated>[A-Za-z_-]+)\s+' +
         r'(?P<when_failed>\S+)\s+(?P<raw_value>\S.*)$')
        attr_keys = ['id', 'flag', 'value', 'worst', 'threshold', 'type',
         'updated', 'when_failed', 'raw_value']

        # Parsing using a state machine, sorry.  We use a state variable, and
        # functions to parse lines in each of the different states.  The function
        # returns the state as a result of reading that line, and we look up the
        # parse function out of an array based on the parse state.
        # parse state 0 = Reading 'key: value' drive information
        # parse state 1 = Reading freeform drive values
        # parse state 2 = Reading attributes information
        # parse state 3 = finished reading.
        parse_state = 0

        # Information section:
        def parse_information(line):
            # Exit parsing information section if we go into the next section
            if line.startswith('=== START OF READ SMART DATA SECTION ==='):
                return 1
            match = info_line_re.search(line)
            if match:
                drive_info['info'][match.group('key')] = match.group('value')
            else:
                # Translate some of the less structured information
                if   line == 'Device does not support SMART':
                    drive_info['info']['SMART support is'] = 'Not supported'
                elif line == 'Device supports SMART and is Enabled':
                    drive_info['info']['SMART support is'] = 'Enabled'
                elif line == 'Error Counter logging not supported':
                    drive_info['info']['Error Counter logging'] = 'Not supported'
                elif line == 'Device does not support Self Test logging':
                    drive_info['info']['Self Test logging'] = 'Not supported'
                elif line == 'Temperature Warning Disabled or Not Supported':
                    drive_info['info']['Temperature Warning'] =\
                        'Disabled or Not Supported'
            return 0

        # Values section:
        def parse_values(line):
            if line.startswith('=== START OF READ SMART DATA SECTION ==='):
                return 1
            if line.startswith('Vendor Specific SMART Attributes with Thresholds'):
                return 2
            if line.startswith('SMART overall-health self-assessment test result'):
                drive_info['health'] = ''.join((line.split(': '))[1:])
                return 1
            # Values section begins with this - ignore:
            if line.startswith('General SMART Values:'):
                return 1

            # Lines starting with a space are continuations of the commentary on
            # the previous setting - ignore
            if len(line) == 0 or line[0] == ' ' or line[0] == "\t":
                return 1
            # Otherwise, join this line to the full line
            if drive_info['full_line']:
                drive_info['full_line'] += ' '
            drive_info['full_line'] += line.strip()

            match = value_line_re.search(drive_info['full_line'])
            if match:
                # Handle the recommended polling time lines, which are joined
                # with the previous line and values are in minutes.
                (key, value) = match.group('key', 'value')
                drive_info['values'][key] = value
                drive_info['full_line'] = ''
            return 1

        # Attributes sections
        def parse_attributes(line):
            if line.startswith('SMART Error Log Version:'):
                return 3
            if line.startswith('Vendor Specific SMART Attributes with Thresholds'):
                return 2
            if len(line) == 0:
                return 2
            match = attr_line_re.match(line)
            if match:
                name = match.group('name')
                drive_info['attributes'][name] = {key: match.group(key)
                    for key in attr_keys}
            return 2

        parse_for_state = [
            parse_information,
            parse_values,
            parse_attributes,
        ]

        for line in content:
            parse_state = parse_for_state[parse_state](line)
            if parse_state == 3:
                break

        # Delete temporary full line storage
        del drive_info['full_line']

        return drive_info
<commit_msg>Further improvements to make SMARTctl more conformant<commit_after>from falafel.core import Mapper
from falafel.core.plugins import mapper

import re

@mapper('smartctl')
class SMARTctl(Mapper):
    """
        Mapper for output of smartctl -a for each drive in system.
        This returns a dict with keys:
         * 'device' - the name of the device after /dev/ - e.g. sda
         * 'info' - the -i info (vendor, product, etc)
         * 'health' - overall health assessment (-H)
         * 'values' - the SMART values (-c) - SMART config on drive firmware
         * 'attributes' - the SMART attributes (-A) - run time data
        This function gets called once per detected file, so we don't have to
        deal with file changes within this process.
    """

    INFO_LINE_RE =  re.compile(r'(?P<key>\w+(?:\s\w+)*):\s+' +
     r'(?P<value>\S.*?)\s*$')
    VALUE_LINE_RE = re.compile(r'(?P<key>\w[A-Za-z _.-]+):\s+' +
     r'\(\s*(?P<value>\S.*?)\)')
    ATTR_LINE_RE =  re.compile(r'^\s*(?P<id>\d+)\s(?P<name>\w+)\s+' +
     r'(?P<flag>0x[0-9a-fA-F]{4})\s+(?P<value>\d{3})\s+' +
     r'(?P<worst>\d{3})\s+(?P<threshold>\d{3})\s+' +
     r'(?P<type>[A-Za-z_-]+)\s+(?P<updated>[A-Za-z_-]+)\s+' +
     r'(?P<when_failed>\S+)\s+(?P<raw_value>\S.*)$')
    ATTR_KEYS = ['id', 'flag', 'value', 'worst', 'threshold', 'type',
     'updated', 'when_failed', 'raw_value']

    def __init__(self, context):
        super(SMARTctl, self).__init__(context)
        filename_re = re.compile(r'smartctl_-a_\.dev\.(?P<device>\w+)$')
        match = filename_re.search(context.path)
        self.data = {}
        if match:
            self.device = match.group('device')
        # Else warn?  fail?

    def parse_content(self, content):
        drive_info = {
            'info'          : {},
            'health'        : 'not parsed',
            'values'        : {},
            'attributes'    : {},
            'full_line'     : '', # hack for persistent line storage
        }

        # Parsing using a state machine, sorry.  We use a state variable, and
        # functions to parse lines in each of the different states.  The function
        # returns the state as a result of reading that line, and we look up the
        # parse function out of an array based on the parse state.
        PARSE_FORMATTED_INFO = 0
        PARSE_FREEFORM_INFO = 1
        PARSE_ATTRIBUTE_INFO = 2
        PARSE_COMPLETE = 3
        parse_state = PARSE_FORMATTED_INFO

        # Information section:
        def parse_information(line):
            # Exit parsing information section if we go into the next section
            if line.startswith('=== START OF READ SMART DATA SECTION ==='):
                return PARSE_FREEFORM_INFO
            match = self.INFO_LINE_RE.search(line)
            if match:
                drive_info['info'][match.group('key')] = match.group('value')
            else:
                # Translate some of the less structured information
                if   line == 'Device does not support SMART':
                    drive_info['info']['SMART support is'] = 'Not supported'
                elif line == 'Device supports SMART and is Enabled':
                    drive_info['info']['SMART support is'] = 'Enabled'
                elif line == 'Error Counter logging not supported':
                    drive_info['info']['Error Counter logging'] = 'Not supported'
                elif line == 'Device does not support Self Test logging':
                    drive_info['info']['Self Test logging'] = 'Not supported'
                elif line == 'Temperature Warning Disabled or Not Supported':
                    drive_info['info']['Temperature Warning'] =\
                        'Disabled or Not Supported'
            return PARSE_FORMATTED_INFO

        # Values section:
        def parse_values(line):
            if line.startswith('=== START OF READ SMART DATA SECTION ==='):
                return PARSE_FREEFORM_INFO
            if line.startswith('Vendor Specific SMART Attributes with Thresholds'):
                return PARSE_ATTRIBUTE_INFO
            if line.startswith('SMART overall-health self-assessment test result'):
                drive_info['health'] = ''.join((line.split(': '))[1:])
                return PARSE_FREEFORM_INFO
            # Values section begins with this - ignore:
            if line.startswith('General SMART Values:'):
                return PARSE_FREEFORM_INFO

            # Lines starting with a space are continuations of the commentary on
            # the previous setting - ignore
            if len(line) == 0 or line[0] == ' ' or line[0] == "\t":
                return PARSE_FREEFORM_INFO
            # Otherwise, join this line to the full line
            if drive_info['full_line']:
                drive_info['full_line'] += ' '
            drive_info['full_line'] += line.strip()

            match = self.VALUE_LINE_RE.search(drive_info['full_line'])
            if match:
                # Handle the recommended polling time lines, which are joined
                # with the previous line and values are in minutes.
                (key, value) = match.group('key', 'value')
                drive_info['values'][key] = value
                drive_info['full_line'] = ''
            return PARSE_FREEFORM_INFO

        # Attributes sections
        def parse_attributes(line):
            if line.startswith('SMART Error Log Version:'):
                return PARSE_COMPLETE
            if line.startswith('Vendor Specific SMART Attributes with Thresholds'):
                return PARSE_ATTRIBUTE_INFO
            if len(line) == 0:
                return PARSE_ATTRIBUTE_INFO
            match = self.ATTR_LINE_RE.match(line)
            if match:
                name = match.group('name')
                drive_info['attributes'][name] = {key: match.group(key)
                    for key in self.ATTR_KEYS}
            return PARSE_ATTRIBUTE_INFO

        parse_for_state = [
            parse_information,
            parse_values,
            parse_attributes,
        ]

        for line in content:
            parse_state = parse_for_state[parse_state](line)
            if parse_state == PARSE_COMPLETE:
                break

        # Delete temporary full line storage
        del drive_info['full_line']

        self.info = drive_info
<|endoftext|>


<commit_before># -*coding: utf-8 -*-
"""Module for creating PosyArray instances.

    Example
    -------
    >>> x = gpkit.Monomial('x')
    >>> px = gpkit.PosyArray([1, x, x**2])

"""

import numpy as np

from . import units as ureg
Quantity = ureg.Quantity

class PosyArray(np.ndarray):
    """A Numpy array with elementwise inequalities and substitutions.

    Parameters
    ----------
    input_array : array-like

    Example
    -------
    >>> px = gpkit.PosyArray([1, x, x**2])
    """

    def __str__(self):
        "Returns list-like string, but with str(el) instead of repr(el)."
        return "[" + ", ".join(str(p) for p in self) + "]"

    def __repr__(self):
        "Returns str(self) tagged with gpkit information."
        return "gpkit.%s(%s)" % (self.__class__.__name__, str(self))

    def __hash__(self):
        return getattr(self, "_hashvalue", np.ndarray.__hash__(self))

    def __new__(cls, input_array):
        "Constructor. Required for objects inheriting from np.ndarray."
        # Input array is an already formed ndarray instance
        # cast to be our class type
        obj = np.asarray(input_array).view(cls)
        return obj

    def __array_finalize__(self, obj):
        "Finalizer. Required for objects inheriting from np.ndarray."
        pass

    def __array_wrap__(self, out_arr, context=None):
        """Called by numpy ufuncs.
        Special case to avoid creation of 0-dimensional arrays
        See http://docs.scipy.org/doc/numpy/user/basics.subclassing.html"""
        if out_arr.ndim:
            return np.ndarray.__array_wrap__(self, out_arr, context)
        return float(out_arr)

    def _latex(self, unused=None, matwrap=True):
        "Returns 1D latex list of contents."
        if len(self.shape) == 1:
            return (("\\begin{bmatrix}" if matwrap else "") +
                    " & ".join(el._latex() for el in self) +
                    ("\\end{bmatrix}" if matwrap else ""))
        elif len(self.shape) == 2:
            return ("\\begin{bmatrix}" +
                    " \\\\\n".join(el._latex(matwrap=False) for el in self) +
                    "\\end{bmatrix}")
        else:
            return None

    def __nonzero__(self):
        "Allows the use of PosyArrays as truth elements."
        return all(p.__nonzero__() for p in self)

    def __bool__(self):
        "Allows the use of PosyArrays as truth elements in python3."
        return all(p.__bool__() for p in self)

    @property
    def c(self):
        try:
            return np.array(self, dtype='float')
        except TypeError:
            raise ValueError("only a posyarray of numbers has a 'c'")

    _eq = np.vectorize(lambda a, b: a == b)

    def __eq__(self, other):
        "Applies == in a vectorized fashion."
        if isinstance(other, Quantity):
            if isinstance(other.magnitude, np.ndarray):
                l = []
                for i, e in enumerate(self):
                    l.append(e == other[i])
                return PosyArray(l)
            else:
                return PosyArray([e == other for e in self])
        return PosyArray(self._eq(self, other))

    def __ne__(self, m):
        "Does type checking, then applies 'not ==' in a vectorized fashion."
        return (not isinstance(other, self.__class__)
                or not all(self._eq(self, other)))

    # inequality constraints
    _leq = np.vectorize(lambda a, b: a <= b)

    def __le__(self, other):
        "Applies '<=' in a vectorized fashion."
        if isinstance(other, Quantity):
            if isinstance(other.magnitude, np.ndarray):
                l = []
                for i, e in enumerate(self):
                    l.append(e <= other[i])
                return PosyArray(l)
            else:
                return PosyArray([e <= other for e in self])
        return PosyArray(self._leq(self, other))

    _geq = np.vectorize(lambda a, b: a >= b)

    def __ge__(self, other):
        "Applies '>=' in a vectorized fashion."
        if isinstance(other, Quantity):
            if isinstance(other.magnitude, np.ndarray):
                l = []
                for i, e in enumerate(self):
                    l.append(e >= other[i])
                return PosyArray(l)
            else:
                return PosyArray([e >= other for e in self])
        return PosyArray(self._geq(self, other))

    def outer(self, other):
        "Returns the array and argument's outer product."
        return PosyArray(np.outer(self, other))

    def sub(self, subs, val=None, require_positive=True):
        "Substitutes into the array"
        return PosyArray([p.sub(subs, val, require_positive) for p in self])

    @property
    def right(self):
        "Self, sampled one index up, with zeropad"
        if self.ndim != 1:
            raise NotImplementedError("not implemented for ndim = %s" %
                                      self.ndim)
        return PosyArray(np.hstack((self[1:], 0)))

    @property
    def left(self):
        "Self, sampled one index down, with zeropad"
        if self.ndim != 1:
            raise NotImplementedError("not implemented for ndim = %s"
                                      % self.ndim)
        return PosyArray(np.hstack((0, self[:-1])))
<commit_msg>correctly cast 0-dimensional arrays, semi-patch for numpy Exception handling<commit_after># -*coding: utf-8 -*-
"""Module for creating PosyArray instances.

    Example
    -------
    >>> x = gpkit.Monomial('x')
    >>> px = gpkit.PosyArray([1, x, x**2])

"""

import numpy as np

from . import units as ureg
Quantity = ureg.Quantity

class PosyArray(np.ndarray):
    """A Numpy array with elementwise inequalities and substitutions.

    Parameters
    ----------
    input_array : array-like

    Example
    -------
    >>> px = gpkit.PosyArray([1, x, x**2])
    """

    def __str__(self):
        "Returns list-like string, but with str(el) instead of repr(el)."
        return "[" + ", ".join(str(p) for p in self) + "]"

    def __repr__(self):
        "Returns str(self) tagged with gpkit information."
        return "gpkit.%s(%s)" % (self.__class__.__name__, str(self))

    def __hash__(self):
        return getattr(self, "_hashvalue", np.ndarray.__hash__(self))

    def __new__(cls, input_array):
        "Constructor. Required for objects inheriting from np.ndarray."
        # Input array is an already formed ndarray instance
        # cast to be our class type
        obj = np.asarray(input_array).view(cls)
        return obj

    def __array_finalize__(self, obj):
        "Finalizer. Required for objects inheriting from np.ndarray."
        pass

    def __array_wrap__(self, out_arr, context=None):
        """Called by numpy ufuncs.
        Special case to avoid creation of 0-dimensional arrays
        See http://docs.scipy.org/doc/numpy/user/basics.subclassing.html"""
        if out_arr.ndim:
            return np.ndarray.__array_wrap__(self, out_arr, context)
        try:
            val = out_arr.item()
            return np.float(val) if isinstance(val, np.generic) else val
        except:
            print("Something went wrong. I'd like to raise a RuntimeWarning,"
                  " but you wouldn't see it because numpy seems to catch all"
                  " Exceptions coming from __array_wrap__.")
            raise

    def _latex(self, unused=None, matwrap=True):
        "Returns 1D latex list of contents."
        if len(self.shape) == 1:
            return (("\\begin{bmatrix}" if matwrap else "") +
                    " & ".join(el._latex() for el in self) +
                    ("\\end{bmatrix}" if matwrap else ""))
        elif len(self.shape) == 2:
            return ("\\begin{bmatrix}" +
                    " \\\\\n".join(el._latex(matwrap=False) for el in self) +
                    "\\end{bmatrix}")
        else:
            return None

    def __nonzero__(self):
        "Allows the use of PosyArrays as truth elements."
        return all(p.__nonzero__() for p in self)

    def __bool__(self):
        "Allows the use of PosyArrays as truth elements in python3."
        return all(p.__bool__() for p in self)

    @property
    def c(self):
        try:
            return np.array(self, dtype='float')
        except TypeError:
            raise ValueError("only a posyarray of numbers has a 'c'")

    _eq = np.vectorize(lambda a, b: a == b)

    def __eq__(self, other):
        "Applies == in a vectorized fashion."
        if isinstance(other, Quantity):
            if isinstance(other.magnitude, np.ndarray):
                l = []
                for i, e in enumerate(self):
                    l.append(e == other[i])
                return PosyArray(l)
            else:
                return PosyArray([e == other for e in self])
        return PosyArray(self._eq(self, other))

    def __ne__(self, m):
        "Does type checking, then applies 'not ==' in a vectorized fashion."
        return (not isinstance(other, self.__class__)
                or not all(self._eq(self, other)))

    # inequality constraints
    _leq = np.vectorize(lambda a, b: a <= b)

    def __le__(self, other):
        "Applies '<=' in a vectorized fashion."
        if isinstance(other, Quantity):
            if isinstance(other.magnitude, np.ndarray):
                l = []
                for i, e in enumerate(self):
                    l.append(e <= other[i])
                return PosyArray(l)
            else:
                return PosyArray([e <= other for e in self])
        return PosyArray(self._leq(self, other))

    _geq = np.vectorize(lambda a, b: a >= b)

    def __ge__(self, other):
        "Applies '>=' in a vectorized fashion."
        if isinstance(other, Quantity):
            if isinstance(other.magnitude, np.ndarray):
                l = []
                for i, e in enumerate(self):
                    l.append(e >= other[i])
                return PosyArray(l)
            else:
                return PosyArray([e >= other for e in self])
        return PosyArray(self._geq(self, other))

    def outer(self, other):
        "Returns the array and argument's outer product."
        return PosyArray(np.outer(self, other))

    def sub(self, subs, val=None, require_positive=True):
        "Substitutes into the array"
        return PosyArray([p.sub(subs, val, require_positive) for p in self])

    @property
    def right(self):
        "Self, sampled one index up, with zeropad"
        if self.ndim != 1:
            raise NotImplementedError("not implemented for ndim = %s" %
                                      self.ndim)
        return PosyArray(np.hstack((self[1:], 0)))

    @property
    def left(self):
        "Self, sampled one index down, with zeropad"
        if self.ndim != 1:
            raise NotImplementedError("not implemented for ndim = %s"
                                      % self.ndim)
        return PosyArray(np.hstack((0, self[:-1])))
<|endoftext|>


<commit_before>#!/mesonet/python/bin/python
# Process the current RAWS ob for IEM processing
# Daryl Herzmann 13 Jun 2003
# 17 Jun 2003	Account for when the file is blank
# 19 Jun 2003	Now we insert into DB instead of silly CSV files
# 30 Jun 2003	Is day == 1 a problem?  I am not sure, since we are already
#		using GMT to initialize mx.DateTime
# 16 Sep 2003	Use iemAccess site stuff

import sys, mx.DateTime, string
from pyIEM import iemdb, iemAccessOb
i = iemdb.iemdb()
mydb = i["other"]
IEM = i["iem"]
mydb.query("SET TIME ZONE 'GMT'")

now = mx.DateTime.gmt()

f = open("%s.txt" % (sys.argv[1],), 'r').readlines()[11:]

if len(f) < 2:
  sys.exit(0)

dl = f[0]

day = int(dl[25:27])
hr  = int(dl[28:30])
mi  = int(dl[30:32])

tmpf = -99
if (string.strip(dl[35:38]) != "MM"):
  tmpf = int(dl[35:38])
dwpf = -99
if (string.strip(dl[39:42]) != "MM"):
  dwpf = int(dl[39:42])
wind = dl[43:47]
pcpn = dl[49:55]
relh = dl[57:60]
peak = dl[67:73]
srad = dl[87:91]

if (wind[2:] == "MM"):
  sknt = "-99"
else:
  sknt = int(wind[2:])
if (wind[:2] == "MM"):
  drct = "-99"
else:
  drct = int(wind[:2]) * 10

#if (day == 1):
#  now = now

now = now + mx.DateTime.RelativeDateTime(second=0, minute=mi, hour=hr)

if (peak[:3] == " MM"):
  gustDir = "-99"
else:
  gustDir = int(peak[:3]) * 10
if (peak[4:] == "MM"):
  gust = "-99"
else:
  gust = peak[4:]

sid = sys.argv[1]

iemob = iemAccessOb.iemAccessOb(sid, 'OT')
iemob.data['tmpf'] = tmpf
iemob.data['sknt'] = sknt
iemob.data['dwpf'] = dwpf
iemob.data['drct'] = drct
iemob.setObTimeGMT(now)
iemob.updateDatabase(IEM)

try:
  mydb.query("INSERT into t%s (station, valid, tmpf, dwpf, sknt, drct) VALUES \
    ('%s', '%s', %s, %s, %s, %s)" % (now.year, sid, now, tmpf, dwpf, sknt, drct) )
except:
  ha = 'ha'
<commit_msg>python2.6<commit_after># Process the current RAWS ob for IEM processing
# Daryl Herzmann 13 Jun 2003
# 17 Jun 2003	Account for when the file is blank
# 19 Jun 2003	Now we insert into DB instead of silly CSV files
# 30 Jun 2003	Is day == 1 a problem?  I am not sure, since we are already
#		using GMT to initialize mx.DateTime
# 16 Sep 2003	Use iemAccess site stuff

import sys
import mx.DateTime
import string
import access
import iemdb
OTHER = iemdb.connect('other', bypass=True)
ocursor = OTHER.cursor()
ocursor.execute("SET TIME ZONE 'GMT'")
IEM = iemdb.connect('iem', bypass=True)
icursor = IEM.cursor()

now = mx.DateTime.gmt()

f = open("%s.txt" % (sys.argv[1],), 'r').readlines()[11:]

if len(f) < 2:
  sys.exit(0)

dl = f[0]

day = int(dl[25:27])
hr  = int(dl[28:30])
mi  = int(dl[30:32])

tmpf = -99
if (string.strip(dl[35:38]) != "MM"):
  tmpf = int(dl[35:38])
dwpf = -99
if (string.strip(dl[39:42]) != "MM"):
  dwpf = int(dl[39:42])
wind = dl[43:47]
pcpn = dl[49:55]
relh = dl[57:60]
peak = dl[67:73]
srad = dl[87:91]

if (wind[2:] == "MM"):
  sknt = "-99"
else:
  sknt = int(wind[2:])
if (wind[:2] == "MM"):
  drct = "-99"
else:
  drct = int(wind[:2]) * 10

#if (day == 1):
#  now = now

now = now + mx.DateTime.RelativeDateTime(second=0, minute=mi, hour=hr)

if (peak[:3] == " MM"):
  gustDir = "-99"
else:
  gustDir = int(peak[:3]) * 10
if (peak[4:] == "MM"):
  gust = "-99"
else:
  gust = peak[4:]

sid = sys.argv[1]

iemob = access.Ob(sid, 'OT')
iemob.data['tmpf'] = tmpf
iemob.data['sknt'] = sknt
iemob.data['dwpf'] = dwpf
iemob.data['drct'] = drct
iemob.setObTimeGMT(now)
iemob.updateDatabase(cursor=icursor)

try:
  ocursor.execute("""INSERT into t%s (station, valid, tmpf, dwpf, sknt, drct) VALUES
    ('%s', '%s', %s, %s, %s, %s)""" % (now.year, sid, now, tmpf, dwpf, sknt, drct) )
except:
  ha = 'ha'

ocursor.close()
icursor.close()
OTHER.commit()
IEM.commit()<|endoftext|>