<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
GeoBase. Rebooted.

    >>> b = NeoBase()
    >>> b.get('ORY', 'city_code_list')
    ['PAR']
    >>> b.get('ORY', 'city_name_list')
    ['Paris']
    >>> b.get('ORY', 'country_code')
    'FR'
    >>> b.distance('ORY', 'CDG')
    34.87...
    >>> b.get_location('ORY')
    LatLng(lat=48.72..., lng=2.35...)
"""

from __future__ import with_statement, print_function, division

from os import getenv
import os.path as op
import operator
from collections import namedtuple
from math import pi, cos, sin, asin, sqrt
import csv
import heapq

__all__ = ['NeoBase', 'LatLng']

_DEF_OPTD_POR_FILE = op.join(op.dirname(__file__), 'optd_por_public.csv')

_DEFAULT_RADIUS = 50

LatLng = namedtuple('LatLng', ['lat', 'lng'])

# Sentinel value for signatures
_sentinel = object()

class NeoBase(object):
    """Main structure, a wrapper around a dict, with dict-like behavior.
    """
    KEY = 0
    FIELDS = (
        ('iata_code', 0, None),
        ('name', 6, None),
        ('lat', 8, None),
        ('lng', 9, None),
        ('country_code', 16, None),
        ('continent_name', 19, None),
        ('timezone', 31, None),
        ('city_code_list', 36, lambda s: s.split(',')),
        ('city_name_list', 37, lambda s: s.split('=')),
        ('location_type', 41, list),
    )
    DUPLICATES = True

    @staticmethod
    def skip(row):
        # This column is the date_until, meaning if this is not empty, the entry is now obsolete
        return bool(row[14])

    def __init__(self, rows=None):
        if rows is None:
            filename = getenv('OPTD_POR_FILE', _DEF_OPTD_POR_FILE)
            with open(filename) as f:
                self._data = self.load(f)
        else:
            self._data = self.load(rows)

    @staticmethod
    def _empty_value():
        return {'__dup__': set()}

    @classmethod
    def load(cls, f):
        """Building a dictionary of geographical data from optd_por.

        >>> path = op.join(op.dirname(__file__), 'optd_por_public.csv')
        >>> with open(path) as f:
        ...     b = NeoBase.load(f)
        >>> b['ORY']['city_code_list']
        ['PAR']
        """
        fields, key_c, duplicates = cls.FIELDS, cls.KEY, cls.DUPLICATES
        empty_value = cls._empty_value

        next(f)  # skipping first line
        data = {}

        for row in csv.reader(f, delimiter='^', quotechar='"'):
            # Comments and empty lines
            if not row or row[0].startswith('#'):
                continue

            if cls.skip(row):
                continue

            d = empty_value()
            for field, c, splitter in fields:
                if splitter is None:
                    d[field] = row[c]
                else:
                    d[field] = splitter(row[c])

            key = row[key_c]
            if key not in data:
                data[key] = d
            elif duplicates:
                prev_d = data[key]
                new_key = '{0}@{1}'.format(key, 1 + len(prev_d['__dup__']))
                data[new_key] = d
                # Exchanging duplicata information
                d['__dup__'] = prev_d['__dup__'] | set([key])
                prev_d['__dup__'].add(new_key)

        return data

    def __iter__(self):
        """Returns iterator of all keys in the base.

        :returns: the iterator of all keys

        >>> b = NeoBase()
        >>> sorted(b)
        ['AAA', 'AAA@1', 'AAB', ...
        """
        return iter(self._data)

    def __contains__(self, key):
        """Test if a key is in the base.

        :param key: the key of to be tested
        :returns:   a boolean

        >>> b = NeoBase()
        >>> 'AN' in b
        False
        >>> 'AGN' in b
        True
        """
        return key in self._data

    def __nonzero__(self):
        """Testing structure emptiness.

        :returns: a boolean

        >>> b = NeoBase()
        >>> if b:
        ...     print('not empty')
        not empty
        """
        return bool(self._data)

    def __len__(self):
        """Testing structure size.

        :returns: a integer

        >>> b = NeoBase()
        >>> 18000 < len(b) < 20000
        True
        """
        return len(self._data)

    def keys(self):
        """Returns iterator of all keys in the base.

        :returns: the iterator of all keys

        >>> b = NeoBase()
        >>> sorted(b.keys())
        ['AAA', 'AAA@1', 'AAB', ...
        """
        return iter(self)

    def set(self, key, **data):
        """Set information.

        >>> b = NeoBase()
        >>> b.get('ORY', 'name')
        'Paris Orly Airport'
        >>> b.set('ORY', name='test')
        >>> b.get('ORY', 'name')
        'test'
        >>> b.set('Wow!', name='test')
        >>> b.get('Wow!', 'name')
        'test'
        """
        if key not in self:
            self._data[key] = self._empty_value()
        self._data[key].update(data)

    def get(self, key, field=None, default=_sentinel):
        """Get data from structure.

        >>> b = NeoBase()
        >>> b.get('OR', 'city_code_list', default=None)
        >>> b.get('ORY', 'city_code_list')
        ['PAR']
        """
        try:
            d = self._data[key]
        except KeyError:
            # Unless default is set, we raise an Exception
            if default is _sentinel:
                raise KeyError("Key not found: {0}".format(key))
            return default

        if field is None:
            return d  # we return the whole dictionary

        try:
            res = d[field]
        except KeyError:
            raise KeyError("Field '{0}' (for key '{1}') not in {2}".format(
                field, key, list(d)))
        else:
            return res

    def get_location(self, key, default=_sentinel):
        """Get None or the geocode.

        >>> b = NeoBase()
        >>> b.get_location('ORY')
        LatLng(lat=48.72..., lng=2.35...)
        """
        if key not in self:
            # Unless default is set, we raise an Exception
            if default is _sentinel:
                raise KeyError("Key not found: {0}".format(key))
            return default

        try:
            loc = LatLng(float(self.get(key, 'lat')),
                         float(self.get(key, 'lng')))

        except (ValueError, TypeError, KeyError):
            # Decode geocode, if error, returns None
            # TypeError : input type is not a string, probably None
            # ValueError: could not convert to float
            # KeyError  : could not find lat or lng 'fields'
            return None
        else:
            return loc

    @staticmethod
    def distance_between_locations(l0, l1):
        """Great circle distance

        :param l0: the LatLng tuple of the first location
        :param l1: the LatLng tuple of the second location
        :returns:  the distance in kilometers

        >>> NeoBase.distance_between_locations((48.84, 2.367), (43.70, 7.26)) # Paris -> Nice
        683.85...

        Case of unknown location.

        >>> NeoBase.distance_between_locations(None, (43.70, 7.26)) # returns None
        """
        if l0 is None or l1 is None:
            return None

        l0_lat = l0[0] / 180 * pi
        l0_lng = l0[1] / 180 * pi
        l1_lat = l1[0] / 180 * pi
        l1_lng = l1[1] / 180 * pi

        # Haversine formula (6371 is Earth radius)
        return 2 * 6371.0 * asin(sqrt(
            sin(0.5 * (l0_lat - l1_lat)) ** 2 +
            sin(0.5 * (l0_lng - l1_lng)) ** 2 *
            cos(l0_lat) * cos(l1_lat)
        ))

    def distance(self, key_0, key_1):
        """Compute distance between two elements.

        This is just a wrapper between the original haversine
        function, but it is probably the most used feature :)

        :param key_0: the first key
        :param key_1: the second key
        :returns:    the distance (km)

        >>> b = NeoBase()
        >>> b.distance('ORY', 'CDG')
        34.87...
        """
        return self.distance_between_locations(self.get_location(key_0),
                                               self.get_location(key_1))

    def _build_distances(self, lat_lng_ref, keys):
        """
        Compute the iterable of (dist, keys) of a reference
        lat_lng and a list of keys. Keys which have not valid
        geocodes will not appear in the results.

        >>> b = NeoBase()
        >>> list(b._build_distances((0,0), ['ORY', 'CDG']))
        [(5422.74..., 'ORY'), (5455.45..., 'CDG')]
        """
        if lat_lng_ref is None:
            return

        for key in keys:
            if key in self:
                lat_lng = self.get_location(key)
                if lat_lng is not None:
                    yield self.distance_between_locations(lat_lng_ref, lat_lng), key

    def find_near_location(self, lat_lng, radius=_DEFAULT_RADIUS, from_keys=None):
        """
        Returns a list of nearby keys from a location (given
        latidude and longitude), and a radius for the search.
        Note that the haversine function, which compute distance
        at the surface of a sphere, here returns kilometers,
        so the radius should be in kms.

        :param lat_lng: the lat_lng of the location
        :param radius:  the radius of the search (kilometers)
        :param from_keys: if None, it takes all keys in consideration, else takes from_keys \
            iterable of keys to perform search.
        :returns:       an iterable of keys (like ['ORY', 'CDG'])

        >>> b = NeoBase()
        >>> # Paris, airports <= 50km
        >>> [b.get(k, 'iata_code') for d, k in sorted(b.find_near_location((48.84, 2.367), 10))]
        ['PAR', 'XGB', 'XHP', 'XPG', 'XEX', 'XTT', 'QAF', 'JDP', 'XBT', 'QNL', 'QBH', 'QFC', 'QEV']
        """
        if from_keys is None:
            from_keys = iter(self)

        for dist, key in self._build_distances(lat_lng, from_keys):
            if dist <= radius:
                yield dist, key

    def find_near(self, key, radius=_DEFAULT_RADIUS, from_keys=None):
        """
        Same as find_near_location, except the location is given
        not by a lat/lng, but with its key, like ORY or SFO.
        We just look up in the base to retrieve lat/lng, and
        call find_near_location.

        :param key:     the key of the location
        :param radius:  the radius of the search (kilometers)
        :param from_keys: if None, it takes all keys in consideration, else takes from_keys \
            iterable of keys to perform search.
        :returns:       a list of keys (like ['ORY', 'CDG'])

        >>> b = NeoBase()
        >>> sorted(b.find_near('ORY', 10)) # Orly, por <= 10km
        [(0.0, 'ORY'), (6.94..., 'XJY'), (9.96..., 'QFC')]
        """
        if from_keys is None:
            from_keys = iter(self)

        if key not in self:
            return

        for dist, key in self.find_near_location(self.get_location(key),
                                                 radius=radius,
                                                 from_keys=from_keys):
            yield dist, key

    def find_closest_from_location(self, lat_lng, N=1, from_keys=None):
        """
        Concept close to find_near_location, but here we do not
        look for the keys radius-close to a location,
        we look for the closest key from this location, given by
        latitude/longitude.

        Note that a similar implementation is done in
        the LocalHelper, to find efficiently N closest location
        in a graph, from a location (using heaps).

        :param lat_lng:   the lat_lng of the location
        :param N:         the N closest results wanted
        :param from_keys: if None, it takes all keys in consideration, else takes from_keys \
            iterable of keys to perform find_closest_from_location. This is useful to combine \
            searches
        :returns:   one key (like 'SFO'), or a list if approximate is not None

        >>> b = NeoBase()
        >>> list(b.find_closest_from_location((43.70, 7.26))) # Nice
        [(0.60..., 'NCE@1')]
        >>> list(b.find_closest_from_location((43.70, 7.26), N=3)) # Nice
        [(0.60..., 'NCE@1'), (5.82..., 'NCE'), (5.89..., 'XBM')]
        """
        if from_keys is None:
            from_keys = iter(self)

        iterable = self._build_distances(lat_lng, from_keys)

        for dist, key in heapq.nsmallest(N, iterable):
            yield dist, key

    def find_with(self, conditions, from_keys=None, reverse=False):
        """Get iterator of all keys with particular field.

        For example, if you want to know all airports in Paris.

        :param conditions: a list of (field, value) conditions
        :param reverse:    we look keys where the field is *not* the particular value
        :returns:          an iterator of matching keys

        Testing several conditions.

        >>> b = NeoBase()
        >>> c0 = [('city_code_list', ['PAR'])]
        >>> c1 = [('location_type', ['H'])]
        >>> len(list(b.find_with(c0)))
        16
        >>> len(list(b.find_with(c0 + c1)))
        2
        """
        if from_keys is None:
            from_keys = iter(self)

        match = operator.ne if reverse else operator.eq

        for key in from_keys:
            if key in self:
                if all(match(self.get(key, f), v) for f, v in conditions):
                    yield key

def main():
    import argparse
    from pprint import pprint

    parser = argparse.ArgumentParser()
    parser.add_argument('keys', nargs='+', help="List of IATA codes")
    args = parser.parse_args()
    b = NeoBase()

    for key in args.keys:
        if key in b:
            pprint(b.get(key))
        else:
            print('{0} not in data.'.format(key))

if __name__ == '__main__':
    main()
<commit_msg>Fix encoding issue on Windows<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
GeoBase. Rebooted.

    >>> b = NeoBase()
    >>> b.get('ORY', 'city_code_list')
    ['PAR']
    >>> b.get('ORY', 'city_name_list')
    ['Paris']
    >>> b.get('ORY', 'country_code')
    'FR'
    >>> b.distance('ORY', 'CDG')
    34.87...
    >>> b.get_location('ORY')
    LatLng(lat=48.72..., lng=2.35...)
"""

from __future__ import with_statement, print_function, division

import sys
PY3 = sys.version_info[0] >= 3
if PY3:
    import io
    from functools import partial
    open_ = partial(io.open, encoding='utf-8')
else:
    open_ = open

from os import getenv
import os.path as op
import operator
from collections import namedtuple
from math import pi, cos, sin, asin, sqrt
import csv
import heapq

__all__ = ['NeoBase', 'LatLng']

_DEF_OPTD_POR_FILE = op.join(op.dirname(__file__), 'optd_por_public.csv')
_DEFAULT_RADIUS = 50
LatLng = namedtuple('LatLng', ['lat', 'lng'])

# Sentinel value for signatures
_sentinel = object()

class NeoBase(object):
    """Main structure, a wrapper around a dict, with dict-like behavior.
    """
    KEY = 0
    FIELDS = (
        ('iata_code', 0, None),
        ('name', 6, None),
        ('lat', 8, None),
        ('lng', 9, None),
        ('country_code', 16, None),
        ('continent_name', 19, None),
        ('timezone', 31, None),
        ('city_code_list', 36, lambda s: s.split(',')),
        ('city_name_list', 37, lambda s: s.split('=')),
        ('location_type', 41, list),
    )
    DUPLICATES = True

    @staticmethod
    def skip(row):
        # This column is the date_until, meaning if this is not empty, the entry is now obsolete
        return bool(row[14])

    def __init__(self, rows=None):
        if rows is None:
            filename = getenv('OPTD_POR_FILE', _DEF_OPTD_POR_FILE)
            with open_(filename) as f:
                self._data = self.load(f)
        else:
            self._data = self.load(rows)

    @staticmethod
    def _empty_value():
        return {'__dup__': set()}

    @classmethod
    def load(cls, f):
        """Building a dictionary of geographical data from optd_por.

        >>> path = op.join(op.dirname(__file__), 'optd_por_public.csv')
        >>> with open_(path) as f:
        ...     b = NeoBase.load(f)
        >>> b['ORY']['city_code_list']
        ['PAR']
        """
        fields, key_c, duplicates = cls.FIELDS, cls.KEY, cls.DUPLICATES
        empty_value = cls._empty_value

        next(f)  # skipping first line
        data = {}

        for row in csv.reader(f, delimiter='^', quotechar='"'):
            # Comments and empty lines
            if not row or row[0].startswith('#'):
                continue

            if cls.skip(row):
                continue

            d = empty_value()
            for field, c, splitter in fields:
                if splitter is None:
                    d[field] = row[c]
                else:
                    d[field] = splitter(row[c])

            key = row[key_c]
            if key not in data:
                data[key] = d
            elif duplicates:
                prev_d = data[key]
                new_key = '{0}@{1}'.format(key, 1 + len(prev_d['__dup__']))
                data[new_key] = d
                # Exchanging duplicata information
                d['__dup__'] = prev_d['__dup__'] | set([key])
                prev_d['__dup__'].add(new_key)

        return data

    def __iter__(self):
        """Returns iterator of all keys in the base.

        :returns: the iterator of all keys

        >>> b = NeoBase()
        >>> sorted(b)
        ['AAA', 'AAA@1', 'AAB', ...
        """
        return iter(self._data)

    def __contains__(self, key):
        """Test if a key is in the base.

        :param key: the key of to be tested
        :returns:   a boolean

        >>> b = NeoBase()
        >>> 'AN' in b
        False
        >>> 'AGN' in b
        True
        """
        return key in self._data

    def __nonzero__(self):
        """Testing structure emptiness.

        :returns: a boolean

        >>> b = NeoBase()
        >>> if b:
        ...     print('not empty')
        not empty
        """
        return bool(self._data)

    def __len__(self):
        """Testing structure size.

        :returns: a integer

        >>> b = NeoBase()
        >>> 18000 < len(b) < 20000
        True
        """
        return len(self._data)

    def keys(self):
        """Returns iterator of all keys in the base.

        :returns: the iterator of all keys

        >>> b = NeoBase()
        >>> sorted(b.keys())
        ['AAA', 'AAA@1', 'AAB', ...
        """
        return iter(self)

    def set(self, key, **data):
        """Set information.

        >>> b = NeoBase()
        >>> b.get('ORY', 'name')
        'Paris Orly Airport'
        >>> b.set('ORY', name='test')
        >>> b.get('ORY', 'name')
        'test'
        >>> b.set('Wow!', name='test')
        >>> b.get('Wow!', 'name')
        'test'
        """
        if key not in self:
            self._data[key] = self._empty_value()
        self._data[key].update(data)

    def get(self, key, field=None, default=_sentinel):
        """Get data from structure.

        >>> b = NeoBase()
        >>> b.get('OR', 'city_code_list', default=None)
        >>> b.get('ORY', 'city_code_list')
        ['PAR']
        """
        try:
            d = self._data[key]
        except KeyError:
            # Unless default is set, we raise an Exception
            if default is _sentinel:
                raise KeyError("Key not found: {0}".format(key))
            return default

        if field is None:
            return d  # we return the whole dictionary

        try:
            res = d[field]
        except KeyError:
            raise KeyError("Field '{0}' (for key '{1}') not in {2}".format(
                field, key, list(d)))
        else:
            return res

    def get_location(self, key, default=_sentinel):
        """Get None or the geocode.

        >>> b = NeoBase()
        >>> b.get_location('ORY')
        LatLng(lat=48.72..., lng=2.35...)
        """
        if key not in self:
            # Unless default is set, we raise an Exception
            if default is _sentinel:
                raise KeyError("Key not found: {0}".format(key))
            return default

        try:
            loc = LatLng(float(self.get(key, 'lat')),
                         float(self.get(key, 'lng')))

        except (ValueError, TypeError, KeyError):
            # Decode geocode, if error, returns None
            # TypeError : input type is not a string, probably None
            # ValueError: could not convert to float
            # KeyError  : could not find lat or lng 'fields'
            return None
        else:
            return loc

    @staticmethod
    def distance_between_locations(l0, l1):
        """Great circle distance

        :param l0: the LatLng tuple of the first location
        :param l1: the LatLng tuple of the second location
        :returns:  the distance in kilometers

        >>> NeoBase.distance_between_locations((48.84, 2.367), (43.70, 7.26)) # Paris -> Nice
        683.85...

        Case of unknown location.

        >>> NeoBase.distance_between_locations(None, (43.70, 7.26)) # returns None
        """
        if l0 is None or l1 is None:
            return None

        l0_lat = l0[0] / 180 * pi
        l0_lng = l0[1] / 180 * pi
        l1_lat = l1[0] / 180 * pi
        l1_lng = l1[1] / 180 * pi

        # Haversine formula (6371 is Earth radius)
        return 2 * 6371.0 * asin(sqrt(
            sin(0.5 * (l0_lat - l1_lat)) ** 2 +
            sin(0.5 * (l0_lng - l1_lng)) ** 2 *
            cos(l0_lat) * cos(l1_lat)
        ))

    def distance(self, key_0, key_1):
        """Compute distance between two elements.

        This is just a wrapper between the original haversine
        function, but it is probably the most used feature :)

        :param key_0: the first key
        :param key_1: the second key
        :returns:    the distance (km)

        >>> b = NeoBase()
        >>> b.distance('ORY', 'CDG')
        34.87...
        """
        return self.distance_between_locations(self.get_location(key_0),
                                               self.get_location(key_1))

    def _build_distances(self, lat_lng_ref, keys):
        """
        Compute the iterable of (dist, keys) of a reference
        lat_lng and a list of keys. Keys which have not valid
        geocodes will not appear in the results.

        >>> b = NeoBase()
        >>> list(b._build_distances((0,0), ['ORY', 'CDG']))
        [(5422.74..., 'ORY'), (5455.45..., 'CDG')]
        """
        if lat_lng_ref is None:
            return

        for key in keys:
            if key in self:
                lat_lng = self.get_location(key)
                if lat_lng is not None:
                    yield self.distance_between_locations(lat_lng_ref, lat_lng), key

    def find_near_location(self, lat_lng, radius=_DEFAULT_RADIUS, from_keys=None):
        """
        Returns a list of nearby keys from a location (given
        latidude and longitude), and a radius for the search.
        Note that the haversine function, which compute distance
        at the surface of a sphere, here returns kilometers,
        so the radius should be in kms.

        :param lat_lng: the lat_lng of the location
        :param radius:  the radius of the search (kilometers)
        :param from_keys: if None, it takes all keys in consideration, else takes from_keys \
            iterable of keys to perform search.
        :returns:       an iterable of keys (like ['ORY', 'CDG'])

        >>> b = NeoBase()
        >>> # Paris, airports <= 50km
        >>> [b.get(k, 'iata_code') for d, k in sorted(b.find_near_location((48.84, 2.367), 10))]
        ['PAR', 'XGB', 'XHP', 'XPG', 'XEX', 'XTT', 'QAF', 'JDP', 'XBT', 'QNL', 'QBH', 'QFC', 'QEV']
        """
        if from_keys is None:
            from_keys = iter(self)

        for dist, key in self._build_distances(lat_lng, from_keys):
            if dist <= radius:
                yield dist, key

    def find_near(self, key, radius=_DEFAULT_RADIUS, from_keys=None):
        """
        Same as find_near_location, except the location is given
        not by a lat/lng, but with its key, like ORY or SFO.
        We just look up in the base to retrieve lat/lng, and
        call find_near_location.

        :param key:     the key of the location
        :param radius:  the radius of the search (kilometers)
        :param from_keys: if None, it takes all keys in consideration, else takes from_keys \
            iterable of keys to perform search.
        :returns:       a list of keys (like ['ORY', 'CDG'])

        >>> b = NeoBase()
        >>> sorted(b.find_near('ORY', 10)) # Orly, por <= 10km
        [(0.0, 'ORY'), (6.94..., 'XJY'), (9.96..., 'QFC')]
        """
        if from_keys is None:
            from_keys = iter(self)

        if key not in self:
            return

        for dist, key in self.find_near_location(self.get_location(key),
                                                 radius=radius,
                                                 from_keys=from_keys):
            yield dist, key

    def find_closest_from_location(self, lat_lng, N=1, from_keys=None):
        """
        Concept close to find_near_location, but here we do not
        look for the keys radius-close to a location,
        we look for the closest key from this location, given by
        latitude/longitude.

        Note that a similar implementation is done in
        the LocalHelper, to find efficiently N closest location
        in a graph, from a location (using heaps).

        :param lat_lng:   the lat_lng of the location
        :param N:         the N closest results wanted
        :param from_keys: if None, it takes all keys in consideration, else takes from_keys \
            iterable of keys to perform find_closest_from_location. This is useful to combine \
            searches
        :returns:   one key (like 'SFO'), or a list if approximate is not None

        >>> b = NeoBase()
        >>> list(b.find_closest_from_location((43.70, 7.26))) # Nice
        [(0.60..., 'NCE@1')]
        >>> list(b.find_closest_from_location((43.70, 7.26), N=3)) # Nice
        [(0.60..., 'NCE@1'), (5.82..., 'NCE'), (5.89..., 'XBM')]
        """
        if from_keys is None:
            from_keys = iter(self)

        iterable = self._build_distances(lat_lng, from_keys)

        for dist, key in heapq.nsmallest(N, iterable):
            yield dist, key

    def find_with(self, conditions, from_keys=None, reverse=False):
        """Get iterator of all keys with particular field.

        For example, if you want to know all airports in Paris.

        :param conditions: a list of (field, value) conditions
        :param reverse:    we look keys where the field is *not* the particular value
        :returns:          an iterator of matching keys

        Testing several conditions.

        >>> b = NeoBase()
        >>> c0 = [('city_code_list', ['PAR'])]
        >>> c1 = [('location_type', ['H'])]
        >>> len(list(b.find_with(c0)))
        16
        >>> len(list(b.find_with(c0 + c1)))
        2
        """
        if from_keys is None:
            from_keys = iter(self)

        match = operator.ne if reverse else operator.eq

        for key in from_keys:
            if key in self:
                if all(match(self.get(key, f), v) for f, v in conditions):
                    yield key

def main():
    import argparse
    from pprint import pprint

    parser = argparse.ArgumentParser()
    parser.add_argument('keys', nargs='+', help="List of IATA codes")
    args = parser.parse_args()
    b = NeoBase()

    for key in args.keys:
        if key in b:
            pprint(b.get(key))
        else:
            print('{0} not in data.'.format(key))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""classes and routines for running nested sampling"""
import random
import numpy as np
import sys
import multiprocessing as mp

class MonteCarloChain(object):
    """Class for doing a Monte Carlo chain

    Parameters
    -----------
    potential : pygmin Potentials
        attribute of system with member function getEnergy (in essence a
        particular potential energy function)
    x : array
        are the coordinates
    takestep : callable takestep object
        take a random montecarlo step, imported from pygmin: takestep(x) makes
        a move from x
    Emax : float
        energy upperbound
    energy : pygmin LJCluster
        energy of a given configuration
    accept_test : list of callables
        it's an array of pointers to functions. The dereferenced functions
        operate a set of tests on the energy/configuration.
    events : list fo callables
        it's an array of pointers to functions. This is general and not
        compulsury e.g. can use if you want to do something with the new
        configuration for the guy.

    Attributes
    ----------
    nsteps : integer
        tot. number of steps
    naccepts : integer
        tot. number of accepted configurations
    xnew : array
        new proposed configuration
    accept : boolean
        true or false if energy constraint is satisfied

    Notes
    -----
    defines parameters for a monte carlo chain from an initial configuration x
    with niter iterations. The step will be accepted subject only to a maximum
    energy criterion and geometric constraints on the configuration.

    See Also
    --------
    NestedSampling
    """
    def __init__(self, potential, x0, takestep, Emax, accept_tests=None, events=None):
        self.potential = potential
        self.x = x0
        self.takestep = takestep
        self.Emax = Emax
        self.energy = self.potential.getEnergy(self.x)
        self.accept_tests = accept_tests

        self.nsteps = 0
        self.naccept = 0

        self.events = events

        if not self.test_configuration(self.x, 0.):
            print "ERROR: initial configuration for monte carlo chain failed configuration test"
            from pygmin.utils.xyz import write_xyz
            with open("error.xyz", "w") as fout:
                write_xyz(fout, self.x)

    def test_configuration(self, x, e):
        if self.accept_tests is not None:
            for test in self.accept_tests:
                if not test(energy=e, coords=x):
                    return False
        return True

    def step(self):
        """
        Do one iteration in the Monte Carlo chain

        Notes
        -----
        copy current configuration x to xnew and then perform a trial move on
        xnew. Accept if the configuration of the new energy is less than Emax
        and if accept_tests are satisfied. If true update x to xnew. Finally,
        optionally applies all events functions to x.
        """
        xnew = self.x.copy()

        # displace configuration
        self.takestep(xnew)

        # get energy
        e = self.potential.getEnergy(xnew)

        accept = e < self.Emax

        if accept:
            accept = self.test_configuration(xnew, e)

        if accept:
            self.x = xnew
            self.energy = e
            self.naccept += 1

        if self.events is not None:
            for event in self.events:
                event(coords=self.x, x=self.x, energy=self.energy, accept=accept)

        self.nsteps += 1

class Replica(object):
    """ Replica is simply a pair of types coordinates (x) and energy"""
    def __init__(self, x, energy):
        self.x = x.copy()
        self.energy = float(energy)

class NestedSampling(object):
    """the main class for implementing nested sampling

    Parameters
    ----------
    system : pygmin System
        is the particular system of interest, say LJCluster
    takestep : callable takestep object
        take a random montecarlo step, imported from pygmin: takestep(x) makes
        a move from x
    mciter : integer
        number of steps in markov chain (sampling)
    accept_test : list of callables
        it's an array of pointers to functions. The dereferenced functions
        operate a set of tests on the energy/configuration.
    mc_runner : callable
        use this object to do the Monte Carlo chain rather than the default

    Attributes
    ----------
    max_energies : list
        array of stored energies (at each step the highest energy configuration
        is stored and replaced by a valid configuration)
    replicas : list
        list of objects of type Replica
        """
    def __init__(self, system, nreplicas, takestep, mciter=100, accept_tests=None, mc_runner=None, nproc=1):
        self.system = system
        self.takestep = takestep
        self.mciter=mciter
        self.accept_tests = accept_tests
        self.mc_runner = mc_runner
        self.nproc = nproc

        self.max_energies = []

        self.setup_replicas(nreplicas)

        self.iter_number = 0

        print "nreplicas", len(self.replicas)
        print "mciter", self.mciter
        sys.stdout.flush()

    def create_replica(self):
        """
        creates a random configuration, evaluates its energy and creates the corresponding Replica object
        """
        x = self.system.get_random_configuration()
        pot = self.system.get_potential()
        e = pot.getEnergy(x)
        return Replica(x, e)

    def sort_replicas(self):
        """
        sorts the replicas in decreasing order of energy
        """
        self.replicas.sort(key=lambda r:r.energy)

    def setup_replicas(self, nreplicas):
        """
        creates nreplicas replicas and sorts them in decreasing order of energy
        """
        self.replicas = [self.create_replica() for i in range(nreplicas)]
        self.sort_replicas()
        print "min replica energy", self.replicas[0].energy
        print "max replica energy", self.replicas[-1].energy

    def adjust_step_size(self, mc):
        """
        Adjust the stepsize to keep the acceptance ratio at a good value

        Notes
        -----
        when ratio naccept/nsteps < target_ratio then decrease step size by a
        correction factor f (new-step-size = old-step-size*f, where 0<f<1),
        else if naccept/nsteps > target_ratio, then increases the step size
        (new-step-size = old-step-size/f, where 0<f<1), although this is
        increased with an upper bound: max_stepsize.
        """
        f = 0.8
        target_ratio = 0.7
        max_stepsize = 0.5
        ratio = float(mc.naccept) / mc.nsteps
        if ratio < target_ratio:
            # reduce step size
            self.takestep.stepsize *= f
        else:
            self.takestep.stepsize /= f
        if self.takestep.stepsize > max_stepsize:
            self.takestep.stepsize = max_stepsize

    def do_monte_carlo_chain(self, x0, Emax, energy=np.nan, **kwargs):
        """
        from an initial configuration do a monte carlo chain with niter iterations,

        the step will be accepted subject only to a maximum energy criterion.
        Re-iterates for mciter steps.  After each step update stepize to meet
        target_ratio.
        """
        if self.mc_runner is None:
            mc = MonteCarloChain(self.system.get_potential(), x0, self.takestep, Emax, accept_tests=self.accept_tests, **kwargs)
            for i in xrange(self.mciter):
                mc.step()
        else:
            mc = self.mc_runner(x0, self.mciter, self.takestep.stepsize, Emax)

        verbose = True
        if verbose:
            # print some data
            dist = np.linalg.norm(mc.x - x0)
            print "step:", self.iter_number, "%accept", float(mc.naccept) / mc.nsteps, \
                "Enew", mc.energy, "Eold", energy, "Emax", Emax, "Emin", self.replicas[0].energy, \
                "stepsize", self.takestep.stepsize, "distance", dist

        if mc.naccept == 0:
            sys.stderr.write("WARNING: zero steps accepted in the Monte Carlo chain %d\n")
            print >> sys.stderr, "WARNING: step:", self.iter_number, "%accept", float(mc.naccept) / mc.nsteps, \
                "Enew", mc.energy, "Eold", energy, "Emax", Emax, "Emin", self.replicas[0].energy, \
                "stepsize", self.takestep.stepsize, "distance", dist

        self.adjust_step_size(mc)
        return mc

    def pop_replica(self):
        """
        remove the replica with the largest energy (last item in the list) and store it in the max_energies array.

        This is then used for integration.
        """
        # pull out the replica with the largest energy
        rmax = self.replicas.pop()

        #remove other nproc-1 replica
        length = self.nproc-1
        for i in range(length):
            self.replicas.pop()

        # add store it for later analysis
        self.max_energies.append(rmax.energy)
        return rmax

    def get_starting_configuration_from_replicas(self):
        # choose a replica randomly
        if self.nproc > 1:
            rstart = random.sample(self.replicas, self.nproc)
            lxstart = []
            lestart = []
            for i in range(len(rstart)):
                lxstart.append(rstart[i].x)
                lestart.append(rstart[i].energy)
            return lxstart, lestart
        else:
            rstart = random.choice(self.replicas)
            return rstart.x, rstart.energy

    def get_starting_configuration(self, Emax):
        return self.get_starting_configuration_from_replicas()

    def add_new_replica(self, x, energy):
        rnew = Replica(x, energy)
        self.replicas.append(rnew)
        return rnew

    def one_iteration_ser(self, xstart, estart, Emax):
        # get a new replica to replace it
        mc = self.do_monte_carlo_chain(xstart, Emax, estart)
        rnew = self.add_new_replica(mc.x, mc.energy)
        self.iter_number += 1
        return 1 ###must return something for parallelisation

    def one_iteration_par(self, xstart, estart, Emax):
        #get nproc new replicas and replace them, use Emax as upper bound
        lEmax = [Emax for i in range(len(xstart))]
        x_tuple = (xstart, estart, lEmax)
        pool = mp.Pool(processes=self.nproc)
        new_func = lambda x:unwrap_one_iteration_ser(self, x)
        result = pool.map(new_func, x_tuple)

    def one_iteration(self):
        rmax = self.pop_replica()
        Emax = rmax.energy
        xstart, estart = self.get_starting_configuration(Emax)

        if self.nproc > 1:
            self.one_iteration_par(xstart, estart, Emax)
        else:
            tmp = self.one_iteration_ser(xstart, estart, Emax)

        self.sort_replicas()

def unwrap_one_iteration_ser(NestedSampling, x_tuple):
    return NestedSampling.one_iteration_ser(x_tuple[0], x_tuple[1], x_tuple[2])

if __name__ == "__main__":
    from lj_run import LJClusterNew
    from pygmin.takestep import RandomDisplacement
    natoms = 13
    nreplicas = 1000
    mciter = 100
    system = LJClusterNew(natoms)

    ns = NestedSampling(system, nreplicas, RandomDisplacement(stepsize=0.5), mciter=mciter)
    for i in range(nreplicas * 300):
        ediff = ns.replicas[-1].energy - ns.replicas[0].energy
        if ediff < .01: break
        ns.one_iteration()
    print ns.max_energies
    print "min replica energy", ns.replicas[0].energy
    print "max replica energy", ns.replicas[-1].energy

    with open("max_energies", "w") as fout:
        fout.write( "\n".join([ str(e) for e in ns.max_energies]) )
        fout.write("\n")

    if True:
        import pylab as pl
        e = np.array(ns.max_energies)
        pl.plot(e)
        pl.show()

<commit_msg>parallelisation in progress<commit_after>"""classes and routines for running nested sampling"""
import random
import numpy as np
import sys
import multiprocessing as mp

class MonteCarloChain(object):
    """Class for doing a Monte Carlo chain

    Parameters
    -----------
    potential : pygmin Potentials
        attribute of system with member function getEnergy (in essence a
        particular potential energy function)
    x : array
        are the coordinates
    takestep : callable takestep object
        take a random montecarlo step, imported from pygmin: takestep(x) makes
        a move from x
    Emax : float
        energy upperbound
    energy : pygmin LJCluster
        energy of a given configuration
    accept_test : list of callables
        it's an array of pointers to functions. The dereferenced functions
        operate a set of tests on the energy/configuration.
    events : list fo callables
        it's an array of pointers to functions. This is general and not
        compulsury e.g. can use if you want to do something with the new
        configuration for the guy.

    Attributes
    ----------
    nsteps : integer
        tot. number of steps
    naccepts : integer
        tot. number of accepted configurations
    xnew : array
        new proposed configuration
    accept : boolean
        true or false if energy constraint is satisfied

    Notes
    -----
    defines parameters for a monte carlo chain from an initial configuration x
    with niter iterations. The step will be accepted subject only to a maximum
    energy criterion and geometric constraints on the configuration.

    See Also
    --------
    NestedSampling
    """
    def __init__(self, potential, x0, takestep, Emax, accept_tests=None, events=None):
        self.potential = potential
        self.x = x0
        self.takestep = takestep
        self.Emax = Emax
        self.energy = self.potential.getEnergy(self.x)
        self.accept_tests = accept_tests

        self.nsteps = 0
        self.naccept = 0

        self.events = events

        if not self.test_configuration(self.x, 0.):
            print "ERROR: initial configuration for monte carlo chain failed configuration test"
            from pygmin.utils.xyz import write_xyz
            with open("error.xyz", "w") as fout:
                write_xyz(fout, self.x)

    def test_configuration(self, x, e):
        if self.accept_tests is not None:
            for test in self.accept_tests:
                if not test(energy=e, coords=x):
                    return False
        return True

    def step(self):
        """
        Do one iteration in the Monte Carlo chain

        Notes
        -----
        copy current configuration x to xnew and then perform a trial move on
        xnew. Accept if the configuration of the new energy is less than Emax
        and if accept_tests are satisfied. If true update x to xnew. Finally,
        optionally applies all events functions to x.
        """
        xnew = self.x.copy()

        # displace configuration
        self.takestep(xnew)

        # get energy
        e = self.potential.getEnergy(xnew)

        accept = e < self.Emax

        if accept:
            accept = self.test_configuration(xnew, e)

        if accept:
            self.x = xnew
            self.energy = e
            self.naccept += 1

        if self.events is not None:
            for event in self.events:
                event(coords=self.x, x=self.x, energy=self.energy, accept=accept)

        self.nsteps += 1

def do_monte_carlo_chain(mc_runner, x0, Emax, energy=np.nan, stepsize, iter_number, replicas, **kwargs):
        """
        from an initial configuration do a monte carlo chain with niter iterations,

        the step will be accepted subject only to a maximum energy criterion.
        Re-iterates for mciter steps.  After each step update stepize to meet
        target_ratio.
        """
        mc = mc_runner(x0, iter_number, stepsize, Emax)

        if self.nproc > 1:
            ####ADD MORE HERE
            x_tuple =
            pool = mp.Pool(processes=self.nproc)
            result = pool.map(mc.parallel_step, x_tuple)
        else:
            for i in xrange(self.mciter):
                mc.step()

        verbose = True
        if verbose:
            # print some data
            dist = np.linalg.norm(mc.x - x0)
            print "step:", self.iter_number, "%accept", float(mc.naccept) / mc.nsteps, \
                "Enew", mc.energy, "Eold", energy, "Emax", Emax, "Emin", self.replicas[0].energy, \
                "stepsize", stepsize, "distance", dist

        if mc.naccept == 0:
            sys.stderr.write("WARNING: zero steps accepted in the Monte Carlo chain %d\n")
            print >> sys.stderr, "WARNING: step:", self.iter_number, "%accept", float(mc.naccept) / mc.nsteps, \
                "Enew", mc.energy, "Eold", energy, "Emax", Emax, "Emin", self.replicas[0].energy, \
                "stepsize", stepsize, "distance", dist

        self.adjust_step_size(mc)
        return mc

class Replica(object):
    """ Replica is simply a pair of types coordinates (x) and energy"""
    def __init__(self, x, energy):
        self.x = x.copy()
        self.energy = float(energy)

class NestedSampling(object):
    """the main class for implementing nested sampling

    Parameters
    ----------
    system : pygmin System
        is the particular system of interest, say LJCluster
    takestep : callable takestep object
        take a random montecarlo step, imported from pygmin: takestep(x) makes
        a move from x
    mciter : integer
        number of steps in markov chain (sampling)
    accept_test : list of callables
        it's an array of pointers to functions. The dereferenced functions
        operate a set of tests on the energy/configuration.
    mc_runner : callable
        use this object to do the Monte Carlo chain rather than the default

    Attributes
    ----------
    max_energies : list
        array of stored energies (at each step the highest energy configuration
        is stored and replaced by a valid configuration)
    replicas : list
        list of objects of type Replica
        """
    def __init__(self, system, nreplicas, takestep, mciter=100, accept_tests=None, mc_runner=None, nproc=1):
        self.system = system
        self.takestep = takestep
        self.mciter=mciter
        self.accept_tests = accept_tests
        self.nproc = nproc

        #choose between compiled and raw version of the mc_runner
        if self.mc_runner is None:
            self.mc_runner = MonteCarloRaw
        else:
            self.mc_runner = mc_runner

        self.max_energies = []

        self.setup_replicas(nreplicas)

        self.iter_number = 0

        print "nreplicas", len(self.replicas)
        print "mciter", self.mciter
        sys.stdout.flush()

    def MonteCarloRaw(self, x0, mciter, stepsize, Emax):
    """non compiled version of MonteCarloRunner"""
        self.takestep.stepsize = stepsize
        return MonteCarloChain(self.system.get_potential(), x0, self.takestep, Emax, accept_tests=self.accept_tests, **kwargs)

    def create_replica(self):
        """
        creates a random configuration, evaluates its energy and creates the corresponding Replica object
        """
        x = self.system.get_random_configuration()
        pot = self.system.get_potential()
        e = pot.getEnergy(x)
        return Replica(x, e)

    def sort_replicas(self):
        """
        sorts the replicas in decreasing order of energy
        """
        self.replicas.sort(key=lambda r:r.energy)

    def setup_replicas(self, nreplicas):
        """
        creates nreplicas replicas and sorts them in decreasing order of energy
        """
        self.replicas = [self.create_replica() for i in range(nreplicas)]
        self.sort_replicas()
        print "min replica energy", self.replicas[0].energy
        print "max replica energy", self.replicas[-1].energy

    def adjust_step_size(self, mc):
        """
        Adjust the stepsize to keep the acceptance ratio at a good value

        Notes
        -----
        when ratio naccept/nsteps < target_ratio then decrease step size by a
        correction factor f (new-step-size = old-step-size*f, where 0<f<1),
        else if naccept/nsteps > target_ratio, then increases the step size
        (new-step-size = old-step-size/f, where 0<f<1), although this is
        increased with an upper bound: max_stepsize.
        """
        f = 0.8
        target_ratio = 0.7
        max_stepsize = 0.5
        ratio = float(mc.naccept) / mc.nsteps
        if ratio < target_ratio:
            # reduce step size
            self.takestep.stepsize *= f
        else:
            self.takestep.stepsize /= f
        if self.takestep.stepsize > max_stepsize:
            self.takestep.stepsize = max_stepsize

    def pop_replica(self):
        """
        remove the replica with the largest energy (last item in the list) and store it in the max_energies array.

        This is then used for integration.
        """
        # pull out the replica with the largest energy
        rmax = self.replicas.pop()

        #remove other nproc-1 replica
        length = self.nproc-1
        for i in range(length):
            self.replicas.pop()

        # add store it for later analysis
        self.max_energies.append(rmax.energy)
        return rmax

    def get_starting_configuration_from_replicas(self):
        # choose a replica randomly
        if self.nproc > 1:
            rstart = random.sample(self.replicas, self.nproc)
            lxstart = []
            lestart = []
            for i in range(len(rstart)):
                lxstart.append(rstart[i].x)
                lestart.append(rstart[i].energy)
            return lxstart, lestart
        else:
            rstart = random.choice(self.replicas)
            return rstart.x, rstart.energy

    def get_starting_configuration(self, Emax):
        return self.get_starting_configuration_from_replicas()

    def add_new_replica(self, x, energy):
        rnew = Replica(x, energy)
        self.replicas.append(rnew)
        return rnew

    def one_iteration_ser(self, xstart, estart, Emax):
        # get a new replica to replace it
        mc = self.do_monte_carlo_chain(xstart, Emax, estart)
        rnew = self.add_new_replica(mc.x, mc.energy)
        self.iter_number += 1
        return 1 ###must return something for parallelisation

    def one_iteration_par(self, xstart, estart, Emax):
        #get nproc new replicas and replace them, use Emax as upper bound
        lEmax = [Emax for i in range(len(xstart))]
        x_tuple = (xstart, estart, lEmax)
        pool = mp.Pool(processes=self.nproc)
        new_func = lambda x:unwrap_one_iteration_ser(self, x)
        result = pool.map(new_func, x_tuple)

    def one_iteration(self):
        rmax = self.pop_replica()
        Emax = rmax.energy
        xstart, estart = self.get_starting_configuration(Emax)

        if self.nproc > 1:
            self.one_iteration_par(xstart, estart, Emax)
        else:
            tmp = self.one_iteration_ser(xstart, estart, Emax)

        self.sort_replicas()

def unwrap_one_iteration_ser(NestedSampling, x_tuple):
    return NestedSampling.one_iteration_ser(x_tuple[0], x_tuple[1], x_tuple[2])

if __name__ == "__main__":
    from lj_run import LJClusterNew
    from pygmin.takestep import RandomDisplacement
    natoms = 13
    nreplicas = 1000
    mciter = 100
    system = LJClusterNew(natoms)

    ns = NestedSampling(system, nreplicas, RandomDisplacement(stepsize=0.5), mciter=mciter)
    for i in range(nreplicas * 300):
        ediff = ns.replicas[-1].energy - ns.replicas[0].energy
        if ediff < .01: break
        ns.one_iteration()
    print ns.max_energies
    print "min replica energy", ns.replicas[0].energy
    print "max replica energy", ns.replicas[-1].energy

    with open("max_energies", "w") as fout:
        fout.write( "\n".join([ str(e) for e in ns.max_energies]) )
        fout.write("\n")

    if True:
        import pylab as pl
        e = np.array(ns.max_energies)
        pl.plot(e)
        pl.show()

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2016 The Shaderc Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Builds the Shaderc project, on Linux, Mac, or Windows.
"""

from __future__ import print_function
import argparse
import os
import platform
import subprocess
import sys

OS = platform.system()

def run(cmd, cwd, env, justprint):
    """Prints a command to run, and optionally runs it.

    Raises a RuntimeError if the command does not launch or otherwise fails.

    Args:
      justprint: If true, then only print the command. Otherwise run the
                   command after printing it.
      cmd:       List of words in the command.
      cwd:       Working directory for the command.
      env:       Environment to pass to subprocess.
    """
    print(cmd)
    if justprint:
        return

    p = subprocess.Popen(cmd, cwd=cwd, env=env)
    (_, _) = p.communicate()
    if p.returncode != 0:
        raise RuntimeError('Failed to run %s in %s' % (cmd, cwd))

def build(args):
    """ Builds Shaderc under specified conditions.

    Args:
        args: An object with attributes:
            srcdir: where Shaderc source can be found
            builddir: build working directory
            installdir: install directory
    """

    if not os.path.isdir(args.srcdir):
        raise RuntimeError('Soure directory %s does not exist' % (args.srcdir))

    # Make paths absolute, and ensure directories exist.
    for d in [args.builddir, args.installdir]:
        if not os.path.isdir(d):
            os.makedirs(d)
    args.srcdir = os.path.abspath(args.srcdir)
    args.builddir = os.path.abspath(args.builddir)
    args.installdir = os.path.abspath(args.installdir)

    print('Building Shaderc:')
    print('   Source     : ', args.srcdir)
    print('   Build dir  : ', args.builddir)
    print('   Install dir: ', args.installdir)
    cmake_command = ['cmake', args.srcdir, '-GNinja',
                     '-DCMAKE_BUILD_TYPE=%s' % args.buildtype,
                     '-DCMAKE_INSTALL_PREFIX=%s' % args.installdir]

    env = None
    if OS == 'Windows':
        p = subprocess.Popen(
            '"%VS140COMNTOOLS%..\\..\\VC\\vcvarsall.bat" & set',
            stdout=subprocess.PIPE, cwd=args.builddir, shell=True)
        env = dict([tuple(line.split('=', 1))
                    for line in p.communicate()[0].splitlines()])
    run(cmake_command, args.builddir, env, args.dry_run)
    run(['ninja', 'install'], args.builddir, env, args.dry_run)
    run(['ctest', '--output-on-failure'], args.builddir, env, args.dry_run)

def main():
    """Builds Shaderc after parsing argument specifying locations of
    files, level of parallelism, and whether it's a dry run that should
    skip actual compilation and installation."""

    parser = argparse.ArgumentParser(description='Build Shaderc simply')
    parser.add_argument('-n', '--dry_run', dest='dry_run', default=False,
                        action='store_true',
                        help='Dry run: Make dirs and only print commands '
                        ' to be run')
    parser.add_argument('--srcdir', dest='srcdir', default='src/shaderc',
                        help='Shaderc source directory. Default "src/shaderc".')
    parser.add_argument('--builddir', dest='builddir', default='out',
                        help='Build directory. Default is "out".')
    parser.add_argument('--installdir', dest='installdir', required=True,
                        help='Installation directory. Required.')
    parser.add_argument('--type', dest='buildtype', default='RelWithDebInfo',
                        help='Build type. Default is RelWithDebInfo')

    arch = None
    if (OS == 'Windows' or OS.startswith('CYGWIN')):
        arch = 'windows-x86'
    if OS == 'Linux':
        arch = 'linux-x86'
    if OS == 'Darwin':
        arch = 'darwin-x86'
    if arch is None:
        raise RuntimeError('Unknown OS: %s' % OS)

    path_default = os.pathsep.join([
        os.path.join(os.getcwd(), 'prebuilts', 'cmake', arch, 'bin'),
        os.path.join(os.getcwd(), 'prebuilts', 'ninja', arch),
        os.path.join(os.getcwd(), 'prebuilts', 'python', arch, 'x64')])

    parser.add_argument('--path', dest='path',
                        default=path_default,
                        help='Extra directories to prepend to the system path, '
                        'separated by your system\'s path delimiter (typically '
                        '":" or ";"). After prepending, path must contain '
                        'cmake, ninja, and python. On Cygwin, the native '
                        'Windows Python must come first. Default is %s.'
                        % path_default)

    args = parser.parse_args()

    if args.path:
        os.environ['PATH'] = os.pathsep.join([args.path, os.getenv('PATH')])

    if OS.startswith('CYGWIN'):
        os.execlp('python', 'python', *sys.argv)  # Escape to Windows.

    build(args)

if __name__ == '__main__':
    main()
<commit_msg>Run cygpath when restarting build.py in Windows.<commit_after>#!/usr/bin/env python

# Copyright 2016 The Shaderc Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Builds the Shaderc project, on Linux, Mac, or Windows.
"""

from __future__ import print_function
import argparse
import os
import platform
import subprocess
import sys

OS = platform.system()

def run(cmd, cwd, env, justprint):
    """Prints a command to run, and optionally runs it.

    Raises a RuntimeError if the command does not launch or otherwise fails.

    Args:
      justprint: If true, then only print the command. Otherwise run the
                   command after printing it.
      cmd:       List of words in the command.
      cwd:       Working directory for the command.
      env:       Environment to pass to subprocess.
    """
    print(cmd)
    if justprint:
        return

    p = subprocess.Popen(cmd, cwd=cwd, env=env)
    (_, _) = p.communicate()
    if p.returncode != 0:
        raise RuntimeError('Failed to run %s in %s' % (cmd, cwd))

def build(args):
    """ Builds Shaderc under specified conditions.

    Args:
        args: An object with attributes:
            srcdir: where Shaderc source can be found
            builddir: build working directory
            installdir: install directory
    """

    if not os.path.isdir(args.srcdir):
        raise RuntimeError('Soure directory %s does not exist' % (args.srcdir))

    # Make paths absolute, and ensure directories exist.
    for d in [args.builddir, args.installdir]:
        if not os.path.isdir(d):
            os.makedirs(d)
    args.srcdir = os.path.abspath(args.srcdir)
    args.builddir = os.path.abspath(args.builddir)
    args.installdir = os.path.abspath(args.installdir)

    print('Building Shaderc:')
    print('   Source     : ', args.srcdir)
    print('   Build dir  : ', args.builddir)
    print('   Install dir: ', args.installdir)
    cmake_command = ['cmake', args.srcdir, '-GNinja',
                     '-DCMAKE_BUILD_TYPE=%s' % args.buildtype,
                     '-DCMAKE_INSTALL_PREFIX=%s' % args.installdir]

    env = None
    if OS == 'Windows':
        p = subprocess.Popen(
            '"%VS140COMNTOOLS%..\\..\\VC\\vcvarsall.bat" & set',
            stdout=subprocess.PIPE, cwd=args.builddir, shell=True)
        env = dict([tuple(line.split('=', 1))
                    for line in p.communicate()[0].splitlines()])
    run(cmake_command, args.builddir, env, args.dry_run)
    run(['ninja', 'install'], args.builddir, env, args.dry_run)
    run(['ctest', '--output-on-failure'], args.builddir, env, args.dry_run)

def cygpath(f):
    p = subprocess.Popen(['cygpath', '-w', f], stdout=subprocess.PIPE)
    return p.communicate()[0].rstrip()

def main():
    """Builds Shaderc after parsing argument specifying locations of
    files, level of parallelism, and whether it's a dry run that should
    skip actual compilation and installation."""

    parser = argparse.ArgumentParser(description='Build Shaderc simply')
    parser.add_argument('-n', '--dry_run', dest='dry_run', default=False,
                        action='store_true',
                        help='Dry run: Make dirs and only print commands '
                        'to be run')
    parser.add_argument('--srcdir', dest='srcdir', default='src/shaderc',
                        help='Shaderc source directory. Default "src/shaderc".')
    parser.add_argument('--builddir', dest='builddir', default='out',
                        help='Build directory. Default is "out".')
    parser.add_argument('--installdir', dest='installdir', required=True,
                        help='Installation directory. Required.')
    parser.add_argument('--type', dest='buildtype', default='RelWithDebInfo',
                        help='Build type. Default is RelWithDebInfo')

    arch = None
    if (OS == 'Windows' or OS.startswith('CYGWIN')):
        arch = 'windows-x86'
    if OS == 'Linux':
        arch = 'linux-x86'
    if OS == 'Darwin':
        arch = 'darwin-x86'
    if arch is None:
        raise RuntimeError('Unknown OS: %s' % OS)

    path_default = os.pathsep.join([
        os.path.join(os.getcwd(), 'prebuilts', 'cmake', arch, 'bin'),
        os.path.join(os.getcwd(), 'prebuilts', 'ninja', arch),
        os.path.join(os.getcwd(), 'prebuilts', 'python', arch, 'x64')])

    parser.add_argument('--path', dest='path',
                        default=path_default,
                        help='Extra directories to prepend to the system path, '
                        'separated by your system\'s path delimiter (typically '
                        '":" or ";"). After prepending, path must contain '
                        'cmake, ninja, and python. On Cygwin, the native '
                        'Windows Python must come first. Default is %s.'
                        % path_default)

    args = parser.parse_args()

    if args.path:
        os.environ['PATH'] = os.pathsep.join([args.path, os.getenv('PATH')])

    if OS.startswith('CYGWIN'):
        # Escape to Windows.
        winargv = []
        args_dict = vars(args)
        for k in args_dict:
            if k=='path' or k.endswith('dir'):
                winargv.extend(['--%s' % k, cygpath(args_dict[k])])
            elif k=='buildtype':
                winargv.extend(['--type', args.buildtype])
            elif k=='dry_run':
                if args.dry_run:
                    winargv.append('-n')
            else:
                winargv.extend(['--%s' % k, args_dict[k]])
        os.execlp('python', 'python', sys.argv[0], *winargv)

    build(args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from django.views.decorators.cache import cache_page
from django.conf import settings
from staticgenerator import StaticGenerator, StaticGeneratorException

def expire_view_cache(request):
    from django.utils.cache import get_cache_key
    key = get_cache_key(request)
    cache.delete(key)

def cache_page_if_anonymous_user(cache_length=60):
    def decorator(func):
        def inner(*args, **kwargs):
            request = args[0]
            if request.user.is_anonymous():
                return cache_page(cache_length)(func)(*args, **kwargs)
            return func(*args, **kwargs)
        return inner

    return decorator

def publish_static(func):
    def inner(*args, **kwargs):
        request = args[0]
        response = func(*args, **kwargs)

        try:
            if ('publish' in kwargs and kwargs['publish']) or (request.user.is_anonymous() and response.publish_static):
                path = request.path
                if not path.endswith('/'):
                    path += '/'
                for trial in range(3):
                    try:
                        StaticGenerator().publish_from_path(path, '', response.content)
                        break
                    except StaticGeneratorException:
                        pass
        except (AttributeError, KeyError) as err:
            pass

        return response

    return inner

def republish_static(path=''):
    import os
    from django.core.urlresolvers import resolve
    from django.http import HttpRequest, QueryDict
    from django.contrib.auth.models import AnonymousUser

    for root, dirs, files in os.walk(os.path.join(settings.WEB_ROOT, path)):
        for name in files:
            filename = os.path.join(root, name)
            relpath = '/'+os.path.relpath(filename, settings.WEB_ROOT)
            relpath = relpath.replace('/index.html', '')
            if relpath == '':
                relpath = '/'

            request = HttpRequest()
            request.path_info = relpath
            request.path = relpath
            request.user = AnonymousUser()

            #request.META.setdefault('SERVER_PORT', 80)
            #request.META.setdefault('SERVER_NAME', self.server_name)

            match = resolve(relpath)
            match.kwargs["publish"] = True
            match.args = (request,) + match.args

            match.func(*(match.args), **(match.kwargs))
<commit_msg>Remove cached files that haven't been accessed for 5 minutes.<commit_after>from django.views.decorators.cache import cache_page
from django.conf import settings
from staticgenerator import StaticGenerator, StaticGeneratorException

def expire_view_cache(request):
    from django.utils.cache import get_cache_key
    key = get_cache_key(request)
    cache.delete(key)

def cache_page_if_anonymous_user(cache_length=60):
    def decorator(func):
        def inner(*args, **kwargs):
            request = args[0]
            if request.user.is_anonymous():
                return cache_page(cache_length)(func)(*args, **kwargs)
            return func(*args, **kwargs)
        return inner

    return decorator

def publish_static(func):
    def inner(*args, **kwargs):
        request = args[0]
        response = func(*args, **kwargs)

        try:
            if ('publish' in kwargs and kwargs['publish']) or (request.user.is_anonymous() and response.publish_static):
                path = request.path
                if not path.endswith('/'):
                    path += '/'
                for trial in range(3):
                    try:
                        StaticGenerator().publish_from_path(path, '', response.content)
                        break
                    except StaticGeneratorException:
                        pass
        except (AttributeError, KeyError) as err:
            pass

        return response

    return inner

def republish_static(path=''):
    import os
    from django.core.urlresolvers import resolve
    from django.http import HttpRequest, QueryDict
    from django.contrib.auth.models import AnonymousUser
    from datetime import datetime

    for root, dirs, files in os.walk(os.path.join(settings.WEB_ROOT, path)):
        for name in files:
            filename = os.path.join(root, name)
            if datetime.now()-datetime.fromtimestamp(os.path.getatime(filename)) > datetime.timedelta(minutes=5):
                # if it hasn't been accessed in the last 5 minutes, delete it instead of republishing
                os.remove(filename)
                os.rmdir(root)
            relpath = '/'+os.path.relpath(filename, settings.WEB_ROOT)
            relpath = relpath.replace('/index.html', '')
            if relpath == '':
                relpath = '/'

            request = HttpRequest()
            request.path_info = relpath
            request.path = relpath
            request.user = AnonymousUser()

            #request.META.setdefault('SERVER_PORT', 80)
            #request.META.setdefault('SERVER_NAME', self.server_name)

            match = resolve(relpath)
            match.kwargs["publish"] = True
            match.args = (request,) + match.args

            match.func(*(match.args), **(match.kwargs))
<|endoftext|>


<commit_before>#!/usr/bin/env python
###############################################################################
#
#	ReportLab Public License Version 1.0
#
#   Except for the change of names the spirit and intention of this
#   license is the same as that of Python
#
#	(C) Copyright ReportLab Inc. 1998-2000.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted, provided
# that the above copyright notice appear in all copies and that both that
# copyright notice and this permission notice appear in supporting
# documentation, and that the name of ReportLab not be used
# in advertising or publicity pertaining to distribution of the software
# without specific, written prior permission.
#
#
# Disclaimer
#
# ReportLab Inc. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS,
# IN NO EVENT SHALL ReportLab BE LIABLE FOR ANY SPECIAL, INDIRECT
# OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
#
###############################################################################
#	$Log: daily.py,v $
#	Revision 1.1  2000/02/23 13:16:56  rgbecker
#	New infrastructure
#
__version__=''' $Id: daily.py,v 1.1 2000/02/23 13:16:56 rgbecker Exp $ '''
'''
script for creating daily cvs archive dump
'''
import os, sys, string, traceback, re

#this is where we extract files etc
groupdir=os.path.normcase(os.path.normpath('/home/groups/ftp/pub/reportlab'))
projdir = os.path.normcase(os.path.normpath('reportlab'))
cvsdir = os.path.join(groupdir,projdir)
release=0		#1 if making a release
USER='rgbecker'

def get_path():
	for i in os.environ.keys():
		if string.upper(i)=='PATH':
			return string.split(os.environ[i],os.pathsep)
	return []

def find_exe(exe):
	if sys.platform=='win32':
		exe='%s.exe'%exe

	for p in get_path():
		f = os.path.join(p,exe)
		if os.path.isfile(f): return f

	return None

def recursive_rmdir(d):
	'destroy directory d'
	if os.path.isdir(d):
		for p in os.listdir(d):
			fn = os.path.join(d,p)
			if os.path.isfile(fn):
				os.remove(fn)
			else:
				recursive_rmdir(fn)
		os.rmdir(d)

def safe_remove(p):
	if os.path.isfile(p): os.remove(p)

def do_exec(cmd, cmdname):
	i=os.popen(cmd,'r')
	print i.read()
	i = i.close()
	if i is not None:
		print 'there was an error executing '+cmdname
		sys.exit(1)

def cvs_checkout(d):
	os.chdir(d)
	recursive_rmdir(cvsdir)

	cvs = find_exe('cvs')
	if cvs is None:
		print "Can't find cvs anywhere on the path"
		os.exit(1)

	os.environ['CVSROOT']=':pserver:%s@cvs1:/cvsroot/reportlab' % USER
	if release:
		do_exec(cvs+' tag %s' % release, 'the tag phase')

	do_exec(cvs+' co reportlab', 'the download phase')

def do_zip(d):
	'create .tgz and .zip file archives of d/reportlab'
	def find_src_files(L,d,N):
		if string.upper(os.path.basename(d))=='CVS': return	#ignore all CVS
		for n in N:
			fn = os.path.normcase(os.path.normpath(os.path.join(d,n)))
			if os.path.isfile(fn): L.append(fn)

	os.chdir(d)
	src_files = []
	os.path.walk(projdir,find_src_files,src_files)
	if release:
		b = "reportlab_%4d%02d%02d" %time.gmtime(time.time())[:3]
	else:
		b = "current"

	tarfile = '%s/%s.tgz' % (groupdir,b)
	zipfile = '%s/%s.zip' % (groupdir,b)
	safe_remove(zipfile)
	safe_remove(tarfile)
	if src_files==[]: return
	src_files = string.join(src_files)

	tar = find_exe('tar')
	do_exec('%s czvf %s %s' % (tar, tarfile, src_files), 'tar creation')

	zip = find_exe('zip')
	do_exec('%s -u %s %s' % (zip, zipfile, src_files), 'zip creation')
	recursive_rmdir(cvsdir)

	if release:
		# make links to the latest outcome
		for b in ['reportlab','current']:
			ltarfile = '%s/%s.tgz' % (groupdir,b)
			lzipfile = '%s/%s.zip' % (groupdir,b)
			safe_remove(lzipfile)
			safe_remove(ltarfile)
			os.symlink(zipfile,lzipfile)
			os.symlink(tarfile,ltarfile)

if __name__=='__main__':
	release = '-release' in sys.argv[1:]
	if release:
		if sys.argv.len()!=3 or sys.argv[1]!='-release':
			print 'Usage:\n    python daily.py [-release tag]'
			sys.exit(1)
		release=sys.argv[2]
	cvs_checkout(groupdir)
	do_zip(groupdir)
<commit_msg>Fixed to release mechanism len()?<commit_after>#!/usr/bin/env python
###############################################################################
#
#	ReportLab Public License Version 1.0
#
#   Except for the change of names the spirit and intention of this
#   license is the same as that of Python
#
#	(C) Copyright ReportLab Inc. 1998-2000.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted, provided
# that the above copyright notice appear in all copies and that both that
# copyright notice and this permission notice appear in supporting
# documentation, and that the name of ReportLab not be used
# in advertising or publicity pertaining to distribution of the software
# without specific, written prior permission.
#
#
# Disclaimer
#
# ReportLab Inc. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS,
# IN NO EVENT SHALL ReportLab BE LIABLE FOR ANY SPECIAL, INDIRECT
# OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
#
###############################################################################
#	$Log: daily.py,v $
#	Revision 1.2  2000/03/28 13:34:26  rgbecker
#	Fixed to release mechanism len()?
#
#	Revision 1.1  2000/02/23 13:16:56  rgbecker
#	New infrastructure
#
__version__=''' $Id: daily.py,v 1.2 2000/03/28 13:34:26 rgbecker Exp $ '''
'''
script for creating daily cvs archive dump
'''
import os, sys, string, traceback, re

#this is where we extract files etc
groupdir=os.path.normcase(os.path.normpath('/home/groups/ftp/pub/reportlab'))
projdir = os.path.normcase(os.path.normpath('reportlab'))
cvsdir = os.path.join(groupdir,projdir)
release=0		#1 if making a release
USER='rgbecker'

def get_path():
	for i in os.environ.keys():
		if string.upper(i)=='PATH':
			return string.split(os.environ[i],os.pathsep)
	return []

def find_exe(exe):
	if sys.platform=='win32':
		exe='%s.exe'%exe

	for p in get_path():
		f = os.path.join(p,exe)
		if os.path.isfile(f): return f

	return None

def recursive_rmdir(d):
	'destroy directory d'
	if os.path.isdir(d):
		for p in os.listdir(d):
			fn = os.path.join(d,p)
			if os.path.isfile(fn):
				os.remove(fn)
			else:
				recursive_rmdir(fn)
		os.rmdir(d)

def safe_remove(p):
	if os.path.isfile(p): os.remove(p)

def do_exec(cmd, cmdname):
	i=os.popen(cmd,'r')
	print i.read()
	i = i.close()
	if i is not None:
		print 'there was an error executing '+cmdname
		sys.exit(1)

def cvs_checkout(d):
	os.chdir(d)
	recursive_rmdir(cvsdir)

	cvs = find_exe('cvs')
	if cvs is None:
		print "Can't find cvs anywhere on the path"
		os.exit(1)

	os.environ['CVSROOT']=':pserver:%s@cvs1:/cvsroot/reportlab' % USER
	if release:
		do_exec(cvs+' tag %s' % release, 'the tag phase')

	do_exec(cvs+' co reportlab', 'the download phase')

def do_zip(d):
	'create .tgz and .zip file archives of d/reportlab'
	def find_src_files(L,d,N):
		if string.upper(os.path.basename(d))=='CVS': return	#ignore all CVS
		for n in N:
			fn = os.path.normcase(os.path.normpath(os.path.join(d,n)))
			if os.path.isfile(fn): L.append(fn)

	os.chdir(d)
	src_files = []
	os.path.walk(projdir,find_src_files,src_files)
	if release:
		b = "reportlab_%4d%02d%02d" %time.gmtime(time.time())[:3]
	else:
		b = "current"

	tarfile = '%s/%s.tgz' % (groupdir,b)
	zipfile = '%s/%s.zip' % (groupdir,b)
	safe_remove(zipfile)
	safe_remove(tarfile)
	if src_files==[]: return
	src_files = string.join(src_files)

	tar = find_exe('tar')
	do_exec('%s czvf %s %s' % (tar, tarfile, src_files), 'tar creation')

	zip = find_exe('zip')
	do_exec('%s -u %s %s' % (zip, zipfile, src_files), 'zip creation')
	recursive_rmdir(cvsdir)

	if release:
		# make links to the latest outcome
		for b in ['reportlab','current']:
			ltarfile = '%s/%s.tgz' % (groupdir,b)
			lzipfile = '%s/%s.zip' % (groupdir,b)
			safe_remove(lzipfile)
			safe_remove(ltarfile)
			os.symlink(zipfile,lzipfile)
			os.symlink(tarfile,ltarfile)

if __name__=='__main__':
	release = '-release' in sys.argv[1:]
	if release:
		if len(sys.argv)!=3 or sys.argv[1]!='-release':
			print 'Usage:\n    python daily.py [-release tag]'
			sys.exit(1)
		release=sys.argv[2]
	cvs_checkout(groupdir)
	do_zip(groupdir)
<|endoftext|>


<commit_before>#!/usr/local/bin/python
###############################################################################
#
#	ReportLab Public License Version 1.0
#
#   Except for the change of names the spirit and intention of this
#   license is the same as that of Python
#
#	(C) Copyright ReportLab Inc. 1998-2000.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted, provided
# that the above copyright notice appear in all copies and that both that
# copyright notice and this permission notice appear in supporting
# documentation, and that the name of ReportLab not be used
# in advertising or publicity pertaining to distribution of the software
# without specific, written prior permission.
#
#
# Disclaimer
#
# ReportLab Inc. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS,
# IN NO EVENT SHALL ReportLab BE LIABLE FOR ANY SPECIAL, INDIRECT
# OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
#
###############################################################################
#	$Log: daily.py,v $
#	Revision 1.21  2000/04/19 15:07:34  rgbecker
#	Syntax error fix
#
#	Revision 1.20  2000/04/19 15:06:28  rgbecker
#	Added py2pdf_dir
#
#	Revision 1.19  2000/04/19 15:00:32  rgbecker
#	pyfontify-->PyFontify
#
#	Revision 1.18  2000/04/19 14:56:07  rgbecker
#	Added CVS_remove
#
#	Revision 1.17  2000/04/19 14:35:04  rgbecker
#	Used -D for py2pdf export
#
#	Revision 1.16  2000/04/19 14:26:13  rgbecker
#	Added tagname
#
#	Revision 1.15  2000/04/19 14:16:07  rgbecker
#	Got rid of userArgs
#
#	Revision 1.14  2000/04/19 14:08:47  rgbecker
#	py2pdf additions
#
#	Revision 1.13  2000/04/07 09:58:10  rgbecker
#	Fixed missing programs problems
#
#	Revision 1.12  2000/04/06 14:10:10  rgbecker
#	Made anonymous
#
#	Revision 1.11  2000/04/06 14:08:00  rgbecker
#	Changed release naming convention
#
#	Revision 1.10  2000/04/06 13:01:58  rgbecker
#	Added import of time
#
#	Revision 1.9  2000/04/06 12:59:48  rgbecker
#	ext-->pserver for release
#
#	Revision 1.8  2000/04/06 12:57:38  rgbecker
#	Print user
#
#	Revision 1.7  2000/04/06 12:54:29  rgbecker
#	Removed rtag again.
#
#	Revision 1.6  2000/04/06 12:50:11  rgbecker
#	Fixed user and rtag
#
#	Revision 1.5  2000/04/06 12:42:33  rgbecker
#	removed attempt at tagging
#
#	Revision 1.4  2000/03/28 13:58:26  rgbecker
#	Add -r tag for release
#
#	Revision 1.3  2000/03/28 13:40:32  rgbecker
#	Fix server for release
#
#	Revision 1.2  2000/03/28 13:34:26  rgbecker
#	Fixed to release mechanism len()?
#
#	Revision 1.1  2000/02/23 13:16:56  rgbecker
#	New infrastructure
#
__version__=''' $Id: daily.py,v 1.21 2000/04/19 15:07:34 rgbecker Exp $ '''
'''
script for creating daily cvs archive dump
'''
import os, sys, string, traceback, re

#this is where we extract files etc
groupdir=os.path.normcase(os.path.normpath('%s/public_ftp'%os.environ['HOME']))
projdir = os.path.normcase(os.path.normpath('reportlab'))
py2pdf_dir = os.path.normcase(os.path.normpath('py2pdf'))
cvsdir = os.path.join(groupdir,projdir)
release=0		#1 if making a release
py2pdf=0		#1 if doing a special py2pdf zip/tgz
#USER=os.environ['USER']
USER='anonymous'

def get_path():
	for i in os.environ.keys():
		if string.upper(i)=='PATH':
			return string.split(os.environ[i],os.pathsep)
	return []

def find_exe(exe):
	if sys.platform=='win32':
		exe='%s.exe'%exe

	for p in get_path():
		f = os.path.join(p,exe)
		if os.path.isfile(f): return f

	print "Can't find %s anywhere on the path" % exe
	return None

def recursive_rmdir(d):
	'destroy directory d'
	if os.path.isdir(d):
		for p in os.listdir(d):
			fn = os.path.join(d,p)
			if os.path.isfile(fn):
				os.remove(fn)
			else:
				recursive_rmdir(fn)
		os.rmdir(d)

def CVS_remove(d):
	'destroy CVS subdirs'
	if os.path.isdir(d):
		if os.split(d)[1]=='CVS':
			recursive_rmdir(d)
		for p in os.listdir(d):
			fn = os.path.join(d,p)
			if os.isdir(fn):
				if p=='CVS':
					recursive_rmdir(fn)
				else:
					CVS_remove(fn)

def safe_remove(p):
	if os.path.isfile(p): os.remove(p)

def do_exec(cmd, cmdname):
	i=os.popen(cmd,'r')
	print i.read()
	i = i.close()
	if i is not None:
		print 'there was an error executing '+cmdname
		sys.exit(1)

def cvs_checkout(d):
	os.chdir(d)
	recursive_rmdir(cvsdir)

	cvs = find_exe('cvs')
	if cvs is None:
		os.exit(1)

	os.environ['CVSROOT']=':pserver:%s@cvs.reportlab.sourceforge.net:/cvsroot/reportlab' % USER
	if release:
		do_exec(cvs+(' export -r %s reportlab' % tagname), 'the export phase')
	else:
		if py2pdf:
			do_exec(cvs+' co reportlab', 'the checkout phase')
			# now we need to move the files & delete those we don't need
			dst = py2pdf_dir
			recursive_rmdir(dst)
			os.mkdir(dst)
			do_exec("mv reportlab/demos/py2pdf/py2pdf.py %s"%dst, "mv py2pdf.py")
			do_exec("mv reportlab/demos/py2pdf/PyFontify.py %s" % dst, "mv pyfontify.py")
			do_exec("rm -r reportlab/demos reportlab/platypus reportlab/lib/styles.py reportlab/README.pdfgen.txt", "rm")
			do_exec("mv reportlab %s" % dst)
			CVS_remove(dst)
		else:
			do_exec(cvs+' co reportlab', 'the checkout phase')

def do_zip(d):
	'create .tgz and .zip file archives of d/reportlab'

	os.chdir(d)
	if release:
		b = tagname
	else:
		b = py2pdf and "py2pdf" or "current"

	tarfile = '%s/%s.tgz' % (groupdir,b)
	zipfile = '%s/%s.zip' % (groupdir,b)

	if py2pdf:
		projdir = py2pdf_dir
		cvsdir = os.path.join(groupdir,projdir)

	tar = find_exe('tar')
	if tar is not None:
		safe_remove(tarfile)
		do_exec('%s czvf %s %s' % (tar, tarfile, projdir), 'tar creation')

	zip = find_exe('zip')
	if zip is not None:
		safe_remove(zipfile)
		do_exec('%s -ur %s %s' % (zip, zipfile, projdir), 'zip creation')
	recursive_rmdir(cvsdir)

	if release:
		# make links to the latest outcome
		for b in ['reportlab','current']:
			ltarfile = '%s/%s.tgz' % (groupdir,b)
			lzipfile = '%s/%s.zip' % (groupdir,b)
			safe_remove(lzipfile)
			safe_remove(ltarfile)
			os.symlink(zipfile,lzipfile)
			os.symlink(tarfile,ltarfile)

if __name__=='__main__':
	def Usage(msg=None):
		if msg is not None: print msg
		print 'Usage:\n    python daily.py [-release tag] | [-py2pdf]'
		sys.exit(1)
	release = '-release' in sys.argv[1:]
	py2pdf = '-py2pdf' in sys.argv[1:]
	if release:
		if py2pdf:
			Usage("Can't have -release and -py2pdf options")
		if len(sys.argv)!=3 or sys.argv[1] != '-release':
			Usage()
		tagname=sys.argv[2]
	cvs_checkout(groupdir)
	do_zip(groupdir)
	if release:
		release = None
		cvs_checkout(groupdir)
		do_zip(groupdir)
<commit_msg>Fixed do_exec call<commit_after>#!/usr/local/bin/python
###############################################################################
#
#	ReportLab Public License Version 1.0
#
#   Except for the change of names the spirit and intention of this
#   license is the same as that of Python
#
#	(C) Copyright ReportLab Inc. 1998-2000.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted, provided
# that the above copyright notice appear in all copies and that both that
# copyright notice and this permission notice appear in supporting
# documentation, and that the name of ReportLab not be used
# in advertising or publicity pertaining to distribution of the software
# without specific, written prior permission.
#
#
# Disclaimer
#
# ReportLab Inc. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS,
# IN NO EVENT SHALL ReportLab BE LIABLE FOR ANY SPECIAL, INDIRECT
# OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
#
###############################################################################
#	$Log: daily.py,v $
#	Revision 1.22  2000/04/19 15:11:58  rgbecker
#	Fixed do_exec call
#
#	Revision 1.21  2000/04/19 15:07:34  rgbecker
#	Syntax error fix
#
#	Revision 1.20  2000/04/19 15:06:28  rgbecker
#	Added py2pdf_dir
#
#	Revision 1.19  2000/04/19 15:00:32  rgbecker
#	pyfontify-->PyFontify
#
#	Revision 1.18  2000/04/19 14:56:07  rgbecker
#	Added CVS_remove
#
#	Revision 1.17  2000/04/19 14:35:04  rgbecker
#	Used -D for py2pdf export
#
#	Revision 1.16  2000/04/19 14:26:13  rgbecker
#	Added tagname
#
#	Revision 1.15  2000/04/19 14:16:07  rgbecker
#	Got rid of userArgs
#
#	Revision 1.14  2000/04/19 14:08:47  rgbecker
#	py2pdf additions
#
#	Revision 1.13  2000/04/07 09:58:10  rgbecker
#	Fixed missing programs problems
#
#	Revision 1.12  2000/04/06 14:10:10  rgbecker
#	Made anonymous
#
#	Revision 1.11  2000/04/06 14:08:00  rgbecker
#	Changed release naming convention
#
#	Revision 1.10  2000/04/06 13:01:58  rgbecker
#	Added import of time
#
#	Revision 1.9  2000/04/06 12:59:48  rgbecker
#	ext-->pserver for release
#
#	Revision 1.8  2000/04/06 12:57:38  rgbecker
#	Print user
#
#	Revision 1.7  2000/04/06 12:54:29  rgbecker
#	Removed rtag again.
#
#	Revision 1.6  2000/04/06 12:50:11  rgbecker
#	Fixed user and rtag
#
#	Revision 1.5  2000/04/06 12:42:33  rgbecker
#	removed attempt at tagging
#
#	Revision 1.4  2000/03/28 13:58:26  rgbecker
#	Add -r tag for release
#
#	Revision 1.3  2000/03/28 13:40:32  rgbecker
#	Fix server for release
#
#	Revision 1.2  2000/03/28 13:34:26  rgbecker
#	Fixed to release mechanism len()?
#
#	Revision 1.1  2000/02/23 13:16:56  rgbecker
#	New infrastructure
#
__version__=''' $Id: daily.py,v 1.22 2000/04/19 15:11:58 rgbecker Exp $ '''
'''
script for creating daily cvs archive dump
'''
import os, sys, string, traceback, re

#this is where we extract files etc
groupdir=os.path.normcase(os.path.normpath('%s/public_ftp'%os.environ['HOME']))
projdir = os.path.normcase(os.path.normpath('reportlab'))
py2pdf_dir = os.path.normcase(os.path.normpath('py2pdf'))
cvsdir = os.path.join(groupdir,projdir)
release=0		#1 if making a release
py2pdf=0		#1 if doing a special py2pdf zip/tgz
#USER=os.environ['USER']
USER='anonymous'

def get_path():
	for i in os.environ.keys():
		if string.upper(i)=='PATH':
			return string.split(os.environ[i],os.pathsep)
	return []

def find_exe(exe):
	if sys.platform=='win32':
		exe='%s.exe'%exe

	for p in get_path():
		f = os.path.join(p,exe)
		if os.path.isfile(f): return f

	print "Can't find %s anywhere on the path" % exe
	return None

def recursive_rmdir(d):
	'destroy directory d'
	if os.path.isdir(d):
		for p in os.listdir(d):
			fn = os.path.join(d,p)
			if os.path.isfile(fn):
				os.remove(fn)
			else:
				recursive_rmdir(fn)
		os.rmdir(d)

def CVS_remove(d):
	'destroy CVS subdirs'
	if os.path.isdir(d):
		if os.split(d)[1]=='CVS':
			recursive_rmdir(d)
		for p in os.listdir(d):
			fn = os.path.join(d,p)
			if os.isdir(fn):
				if p=='CVS':
					recursive_rmdir(fn)
				else:
					CVS_remove(fn)

def safe_remove(p):
	if os.path.isfile(p): os.remove(p)

def do_exec(cmd, cmdname):
	i=os.popen(cmd,'r')
	print i.read()
	i = i.close()
	if i is not None:
		print 'there was an error executing '+cmdname
		sys.exit(1)

def cvs_checkout(d):
	os.chdir(d)
	recursive_rmdir(cvsdir)

	cvs = find_exe('cvs')
	if cvs is None:
		os.exit(1)

	os.environ['CVSROOT']=':pserver:%s@cvs.reportlab.sourceforge.net:/cvsroot/reportlab' % USER
	if release:
		do_exec(cvs+(' export -r %s %s' % (tagname,projdir)), 'the export phase')
	else:
		do_exec(cvs+' co %s' % projdir, 'the checkout phase')
		if py2pdf:
			# now we need to move the files & delete those we don't need
			dst = py2pdf_dir
			recursive_rmdir(dst)
			os.mkdir(dst)
			do_exec("mv reportlab/demos/py2pdf/py2pdf.py %s"%dst, "mv py2pdf.py")
			do_exec("mv reportlab/demos/py2pdf/PyFontify.py %s" % dst, "mv pyfontify.py")
			do_exec("rm -r reportlab/demos reportlab/platypus reportlab/lib/styles.py reportlab/README.pdfgen.txt", "rm")
			do_exec("mv %s %s" % (projdir,dst), "moving %s to %s" %(projdir,py2pdf_dir))
			CVS_remove(dst)

def do_zip(d):
	'create .tgz and .zip file archives of d/reportlab'

	os.chdir(d)
	if release:
		b = tagname
	else:
		b = py2pdf and "py2pdf" or "current"

	tarfile = '%s/%s.tgz' % (groupdir,b)
	zipfile = '%s/%s.zip' % (groupdir,b)

	if py2pdf:
		projdir = py2pdf_dir
		cvsdir = os.path.join(groupdir,projdir)

	tar = find_exe('tar')
	if tar is not None:
		safe_remove(tarfile)
		do_exec('%s czvf %s %s' % (tar, tarfile, projdir), 'tar creation')

	zip = find_exe('zip')
	if zip is not None:
		safe_remove(zipfile)
		do_exec('%s -ur %s %s' % (zip, zipfile, projdir), 'zip creation')
	recursive_rmdir(cvsdir)

	if release:
		# make links to the latest outcome
		for b in ['reportlab','current']:
			ltarfile = '%s/%s.tgz' % (groupdir,b)
			lzipfile = '%s/%s.zip' % (groupdir,b)
			safe_remove(lzipfile)
			safe_remove(ltarfile)
			os.symlink(zipfile,lzipfile)
			os.symlink(tarfile,ltarfile)

if __name__=='__main__':
	def Usage(msg=None):
		if msg is not None: print msg
		print 'Usage:\n    python daily.py [-release tag] | [-py2pdf]'
		sys.exit(1)
	release = '-release' in sys.argv[1:]
	py2pdf = '-py2pdf' in sys.argv[1:]
	if release:
		if py2pdf:
			Usage("Can't have -release and -py2pdf options")
		if len(sys.argv)!=3 or sys.argv[1] != '-release':
			Usage()
		tagname=sys.argv[2]
	cvs_checkout(groupdir)
	do_zip(groupdir)
	if release:
		release = None
		cvs_checkout(groupdir)
		do_zip(groupdir)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

import uuid

from flask import request, url_for
from flask.ext.babel import gettext
from werkzeug.urls import Href, url_unquote

from api import jsonify
from models.assembly import term as assembly_term
from models.election import current_assembly_id

breadcrumb_home = ['home']

def init_app(app):
    app.jinja_env.add_extension('jinja2.ext.loopcontrols')
    app.jinja_env.filters['jsonify'] = jsonify
    app.jinja_env.globals.update(
        hasattr=hasattr,
        int=int,
        max=max,
        min=min,
        zip=zip,
    )
    app.jinja_env.globals.update(
        app=app,
        assembly_term=assembly_term,
        current_assembly_id=current_assembly_id,
        url_for_query=url_for_query,
        url_for_noencode=url_for_noencode,
    )

    app.breadcrumbed_views = {}
    @app.context_processor
    def inject_breadcrumb():
        if request.endpoint not in app.breadcrumbed_views:
            return {}

        breadcrumbs = app.breadcrumbed_views.get(request.endpoint, [])
        return dict(breadcrumbs=breadcrumbs)

def guid_factory():
    guids = {}

    def factory(key):
        val = guids.get(key)
        if not val:
            val = 'u%s' % uuid.uuid4().hex[:8]
            guids[key] = val
        return val

    return factory

def url_for_query(**kwargs):
    h = Href(request.base_url)
    args = dict(request.args, **kwargs)
    return h(**args)

def url_for_noencode(endpoint, **kwargs):
    return url_unquote(url_for(endpoint, **kwargs))

def breadcrumb(app, *hierarchy):
    def decorator(f):
        app.breadcrumbed_views[f.__name__] = breadcrumb_home + list(hierarchy)
        return f
    return decorator
<commit_msg>Fix import path bug caused by merging api branch<commit_after># -*- encoding: utf-8 -*-

import uuid

from flask import request, url_for
from flask.ext.babel import gettext
from werkzeug.urls import Href, url_unquote

from jsonify import jsonify
from models.assembly import term as assembly_term
from models.election import current_assembly_id

breadcrumb_home = ['home']

def init_app(app):
    app.jinja_env.add_extension('jinja2.ext.loopcontrols')
    app.jinja_env.filters['jsonify'] = jsonify
    app.jinja_env.globals.update(
        hasattr=hasattr,
        int=int,
        max=max,
        min=min,
        zip=zip,
    )
    app.jinja_env.globals.update(
        app=app,
        assembly_term=assembly_term,
        current_assembly_id=current_assembly_id,
        url_for_query=url_for_query,
        url_for_noencode=url_for_noencode,
    )

    app.breadcrumbed_views = {}
    @app.context_processor
    def inject_breadcrumb():
        if request.endpoint not in app.breadcrumbed_views:
            return {}

        breadcrumbs = app.breadcrumbed_views.get(request.endpoint, [])
        return dict(breadcrumbs=breadcrumbs)

def guid_factory():
    guids = {}

    def factory(key):
        val = guids.get(key)
        if not val:
            val = 'u%s' % uuid.uuid4().hex[:8]
            guids[key] = val
        return val

    return factory

def url_for_query(**kwargs):
    h = Href(request.base_url)
    args = dict(request.args, **kwargs)
    return h(**args)

def url_for_noencode(endpoint, **kwargs):
    return url_unquote(url_for(endpoint, **kwargs))

def breadcrumb(app, *hierarchy):
    def decorator(f):
        app.breadcrumbed_views[f.__name__] = breadcrumb_home + list(hierarchy)
        return f
    return decorator
<|endoftext|>