<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

from .worker import worker as w, logger

import gevent
import gevent.queue
import requests
import time
from datetime import datetime, timedelta
import functools
from collections import defaultdict
from translate import Translator

from .config import GITHUB_CRENDENTIALS
from .db import mongodb, redis, format_key as _format
from .geo import geo_info
from .fetch import fetch_one, events_process, file_process, events_process_lang_contrib

ghapi_url = "https://api.github.com/users/{username}"
geoname_url = "http://api.geonames.org/search"

def concurrency(n):
    '''no more than n processes running.'''
    def wrapper(fn):
        @functools.wraps(fn)
        def wrap(*args, **kwargs):
            r = redis()
            key = _format('worker:concurrency')
            try:
                if r.hincrby(key, fn.__name__, 1) > n:
                    logger.info("No more tasks for %s..." % fn.__name__)
                else:
                    return fn(*args, **kwargs)
            except:
                pass
            finally:
                r.hincrby(key, fn.__name__, -1)
        return wrap
    return wrapper

@w.task
def update_user(username):
    '''update user's info from github'''
    logger.info("Updating %s" % username)
    while not _update_user(username):
        logger.info("Holding github user task for 10 minutes.")
        time.sleep(10 * 60)

def _update_user(username):
    username = username.lower()
    users = mongodb().users_stats
    user = users.find_one({"_id": username}, {"info": 1})

    successful = True
    info = user.get('info', {}) if user else {}
    etag = info.get("etag", None)
    location = info.get("location", None)

    r = None
    try:
        # Work out the authentication headers.
        auth = {}
        client_id, client_secret = GITHUB_CRENDENTIALS.split(":")
        if client_id is not None and client_secret is not None:
            auth["client_id"] = client_id
            auth["client_secret"] = client_secret

        # Perform a conditional fetch on the database.
        headers = {"If-None-Match": etag} if etag else {}
        r = requests.get(ghapi_url.format(username=username), params=auth,
                         headers=headers)
        code = r.status_code
        if code == requests.codes.ok:
            data = r.json()
            data['etag'] = r.headers["ETag"]
            users.update({"_id": username},
                         {"$set": {"info": data}},
                         upsert=True)
            location = data.get('location', None)
        elif code == 403:
            logger.info("*** Limitation reached.")
            successful = False
    except:
        pass
    finally:
        if r:
            r.close()
    if location not in [None, ""]:
        if not mongodb().locations.find_one({"_id": location.lower()}):
            update_location.delay(location)
    return successful

@w.task
def update_location(location):
    '''get location data(contry, city...) from location name'''
    if location in [None, ""]:
        return

    logger.info("Retrieving location %s" % location)
    location = location.lower()
    locations = mongodb().locations
    if locations.find_one({"_id": location}):
        return  # return if the loc is already in the db
    loc = {"_id": location}
    loc.update(geo_info(location) or {})
    locations.update({"_id": location}, loc, True)

@w.task(ignore_result=True)
@concurrency(1)
def update_all_users():
    '''Traverse all users and retrieve its info from github.'''
    def _worker(q):
        for name in q:
            try:
                update_user.s(name).delay().get()
            except Exception as e:
                logger.error(e)

    index, count, r = 0, 100, redis()
    q = gevent.queue.Queue(count)
    workers = [gevent.spawn(_worker, q) for i in range(count)]
    while True:
        names = r.zrevrange(_format("user"), index, index + count)
        for name in names:
            q.put(name)
        if len(names) < count:
            break
        index += count
        logger.info("Updated %d users." % index)
    for i in range(len(workers)):
        q.put(StopIteration)
    gevent.joinall(workers)

@w.task(ignore_result=True)
@concurrency(1)
def fetch_timeline(year=2012, month=3, day=1, threads=4):
    '''worker process to go through all timeline data since 2012/3/1.'''
    def _worker(q):
        # worker process.
        for year, month, day, hour in q:
            fetch_worker.s(year, month, day, hour).delay().get()
    q = gevent.queue.Queue(8)
    workers = [gevent.spawn(_worker, q) for i in range(threads)]
    since = datetime(year, month, day)
    while since < datetime.today() - timedelta(days=1):
        q.put([since.year, since.month, since.day, since.hour])
        since += timedelta(hours=1)
    [q.put(StopIteration) for w in workers]
    gevent.joinall(workers)

@w.task
def fetch_worker(year, month, day, hour):
    '''fetch one hour's timeline data and save it to db.'''
    try:
        file_process(fetch_one(year, month, day, hour), [events_process, events_process_lang_contrib])
    except Exception as e:
        logger.error("Error during processing %d-%d-%d %d hr: %s" % (year, month, day, hour, e))

@w.task
@concurrency(1)
def country_rank():
    '''Activities per country and month.'''
    default = lambda: {'year': defaultdict(int), 'month': defaultdict(lambda: defaultdict(int)), 'total': 0}
    countries = defaultdict(default)
    for user in mongodb().users_stats.find({'loc.country': {'$ne': None}},
                                           {'month': 1, 'loc.country': 1, 'contrib': 1}):
        country = user['loc']['country']
        if country not in countries:
            countries[country]['users'] = 0
            countries[country]['contrib'] = defaultdict(default)
            countries[country]['display'] = {'en': country}
        countries[country]['users'] += 1
        for year in user.get('month', {}):
            for month in user['month'][year]:
                countries[country]['month'][year][month] += user['month'][year][month]
                countries[country]['year'][year] += user['month'][year][month]
                countries[country]['total'] += user['month'][year][month]
        cont = countries[country]['contrib']
        for lang in user.get('contrib', {}):
            for year in user['contrib'][lang]:
                for month in user['contrib'][lang][year]:
                    cont[lang]['month'][year][month] += user['contrib'][lang][year][month]
                    cont[lang]['year'][year] += user['contrib'][lang][year][month]
                    cont[lang]['total'] += user['contrib'][lang][year][month]
    results = {country: translate.delay(country, to_lang='zh') for country in countries}
    for country, result in results.items():
        try:
            countries[country]['display']['zh'] = result.get()
        except:
            logger.error("Error during translating %s." % country)
    stats = mongodb().country_stats
    for country in countries:
        stats.update({'_id': country}, {'$set': countries[country]}, True)

@w.task
@concurrency(1)
def city_rank():
    '''Activities per city and month.'''
    localities = {}
    default = lambda: {'year': defaultdict(int), 'month': defaultdict(lambda: defaultdict(int)), 'total': 0}
    for user in mongodb().users_stats.find({'loc.city': {'$ne': None}},
                                           {'month': 1, 'loc': 1, 'contrib': 1}):
        country, state, city = [user['loc'].get(t, None) for t in ['country', 'state', 'city']]
        if city not in localities:
            localities[city] = default()
            localities[city]['users'] = 0
            localities[city]['contrib'] = defaultdict(default)
            localities[city]['country'] = country
            localities[city]['state'] = state
            localities[city]['display'] = {'en': city}
        localities[city]['users'] += 1
        for year in user.get('month', {}):
            for month in user['month'][year]:
                localities[city]['month'][year][month] += user['month'][year][month]
                localities[city]['year'][year] += user['month'][year][month]
                localities[city]['total'] += user['month'][year][month]
        cont = localities[city]['contrib']
        for lang in user.get('contrib', {}):
            for year in user['contrib'][lang]:
                for month in user['contrib'][lang][year]:
                    cont[lang]['month'][year][month] += user['contrib'][lang][year][month]
                    cont[lang]['year'][year] += user['contrib'][lang][year][month]
                    cont[lang]['total'] += user['contrib'][lang][year][month]
    results = {city: translate.delay(city, to_lang='zh') for city in localities}
    for city, result in results.items():
        try:
            localities[city]['display']['zh'] = result.get()
        except:
            logger.error("Error during translating %s." % city)
    stats = mongodb().city_stats
    for city, value in localities.items():
        stats.update({'_id': city}, {'$set': value}, True)

@w.task
def translate(text, to_lang='zh'):
    translation = mongodb().translation
    t = translation.find_one({'_id': text, to_lang: {'$ne': None}}, {to_lang: 1})
    result = t and t[to_lang]
    if not result:
        result = Translator(to_lang=to_lang, from_lang='en').translate(text.encode('utf8'))
        translation.update({'_id': text}, {'$set': {to_lang: result}}, True)
    return result

@w.task
@concurrency(1)
def update_user_location():
    locs = {}
    for location in mongodb().locations.find({},
                                             {'locality.long_name': 1,
                                              'country.long_name': 1,
                                              'administrative_area_level_1.long_name': 1
                                              }):
        locs[location['_id']] = {
            'country': location.get('country', {}).get('long_name', None),
            'state': location.get('administrative_area_level_1', {}).get('long_name', None),
            'city': location.get('locality', {}).get('long_name', None)
        }
    users_stats = mongodb().users_stats
    for user in users_stats.find({'info.location': {'$ne': None}, 'loc': None}):
        location = user['info']['location'].lower()
        if location in locs:
            users_stats.update({'_id': user['_id']}, {'$set': {'loc': locs[location]}})
    country_rank.delay().get()
    city_rank.delay().get()

# @w.task
# def lang_rank(lang, year=None):
#     year = year or datetime.now().year
#     key = 'contrib.%s.%d' % (lang, year)
#     users = {}
#     for user in mongodb().users_stats.find({key: {'$ne': None}}, {key: 1}):
#         data = user['contrib'][lang]['%d' % year]
#         users[user['_id']] = sum([data[month] for month in data])
<commit_msg>add timeout<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

from .worker import worker as w, logger

import gevent
import gevent.queue
import requests
import time
from datetime import datetime, timedelta
import functools
from collections import defaultdict
from translate import Translator

from .config import GITHUB_CRENDENTIALS
from .db import mongodb, redis, format_key as _format
from .geo import geo_info
from .fetch import fetch_one, events_process, file_process, events_process_lang_contrib

ghapi_url = "https://api.github.com/users/{username}"
geoname_url = "http://api.geonames.org/search"

def concurrency(n):
    '''no more than n processes running.'''
    def wrapper(fn):
        @functools.wraps(fn)
        def wrap(*args, **kwargs):
            r = redis()
            key = _format('worker:concurrency')
            try:
                if r.hincrby(key, fn.__name__, 1) > n:
                    logger.info("No more tasks for %s..." % fn.__name__)
                else:
                    return fn(*args, **kwargs)
            except:
                pass
            finally:
                r.hincrby(key, fn.__name__, -1)
        return wrap
    return wrapper

@w.task
def update_user(username):
    '''update user's info from github'''
    logger.info("Updating %s" % username)
    while not _update_user(username):
        logger.info("Holding github user task for 10 minutes.")
        time.sleep(10 * 60)

def _update_user(username):
    username = username.lower()
    users = mongodb().users_stats
    user = users.find_one({"_id": username}, {"info": 1})

    successful = True
    info = user.get('info', {}) if user else {}
    etag = info.get("etag", None)
    location = info.get("location", None)

    r = None
    try:
        # Work out the authentication headers.
        auth = {}
        client_id, client_secret = GITHUB_CRENDENTIALS.split(":")
        if client_id is not None and client_secret is not None:
            auth["client_id"] = client_id
            auth["client_secret"] = client_secret

        # Perform a conditional fetch on the database.
        headers = {"If-None-Match": etag} if etag else {}
        r = requests.get(ghapi_url.format(username=username), params=auth,
                         headers=headers)
        code = r.status_code
        if code == requests.codes.ok:
            data = r.json()
            data['etag'] = r.headers["ETag"]
            users.update({"_id": username},
                         {"$set": {"info": data}},
                         upsert=True)
            location = data.get('location', None)
        elif code == 403:
            logger.info("*** Limitation reached.")
            successful = False
    except:
        pass
    finally:
        if r:
            r.close()
    if location not in [None, ""]:
        if not mongodb().locations.find_one({"_id": location.lower()}):
            update_location.delay(location)
    return successful

@w.task
def update_location(location):
    '''get location data(contry, city...) from location name'''
    if location in [None, ""]:
        return

    logger.info("Retrieving location %s" % location)
    location = location.lower()
    locations = mongodb().locations
    if locations.find_one({"_id": location}):
        return  # return if the loc is already in the db
    loc = {"_id": location}
    loc.update(geo_info(location) or {})
    locations.update({"_id": location}, loc, True)

@w.task(ignore_result=True)
@concurrency(1)
def update_all_users():
    '''Traverse all users and retrieve its info from github.'''
    def _worker(q):
        for name in q:
            try:
                update_user.s(name).delay().get()
            except Exception as e:
                logger.error(e)

    index, count, r = 0, 100, redis()
    q = gevent.queue.Queue(count)
    workers = [gevent.spawn(_worker, q) for i in range(count)]
    while True:
        names = r.zrevrange(_format("user"), index, index + count)
        for name in names:
            q.put(name)
        if len(names) < count:
            break
        index += count
        logger.info("Updated %d users." % index)
    for i in range(len(workers)):
        q.put(StopIteration)
    gevent.joinall(workers)

@w.task(ignore_result=True)
@concurrency(1)
def fetch_timeline(year=2012, month=3, day=1, threads=4):
    '''worker process to go through all timeline data since 2012/3/1.'''
    def _worker(q):
        # worker process.
        for year, month, day, hour in q:
            try:
                fetch_worker.delay(year, month, day, hour).get()
            except:
                logger.error('Error during fetching %d-%d-%d %d.' % (year, month, day, hour))
    q = gevent.queue.Queue(8)
    workers = [gevent.spawn(_worker, q) for i in range(threads)]
    since = datetime(year, month, day)
    while since < datetime.today() - timedelta(days=1):
        q.put([since.year, since.month, since.day, since.hour])
        since += timedelta(hours=1)
    [q.put(StopIteration) for w in workers]
    gevent.joinall(workers)

@w.task(time_limit=3600*4)
def fetch_worker(year, month, day, hour):
    '''fetch one hour's timeline data and save it to db.'''
    try:
        file_process(fetch_one(year, month, day, hour), [events_process, events_process_lang_contrib])
    except Exception as e:
        logger.error("Error during processing %d-%d-%d %d hr: %s" % (year, month, day, hour, e))

@w.task(time_limit=3600*4)
@concurrency(1)
def country_rank():
    '''Activities per country and month.'''
    default = lambda: {'year': defaultdict(int), 'month': defaultdict(lambda: defaultdict(int)), 'total': 0}
    countries = defaultdict(default)
    for user in mongodb().users_stats.find({'loc.country': {'$ne': None}},
                                           {'month': 1, 'loc.country': 1, 'contrib': 1}):
        country = user['loc']['country']
        if country not in countries:
            countries[country]['users'] = 0
            countries[country]['contrib'] = defaultdict(default)
            countries[country]['display'] = {'en': country}
        countries[country]['users'] += 1
        for year in user.get('month', {}):
            for month in user['month'][year]:
                countries[country]['month'][year][month] += user['month'][year][month]
                countries[country]['year'][year] += user['month'][year][month]
                countries[country]['total'] += user['month'][year][month]
        cont = countries[country]['contrib']
        for lang in user.get('contrib', {}):
            for year in user['contrib'][lang]:
                for month in user['contrib'][lang][year]:
                    cont[lang]['month'][year][month] += user['contrib'][lang][year][month]
                    cont[lang]['year'][year] += user['contrib'][lang][year][month]
                    cont[lang]['total'] += user['contrib'][lang][year][month]
    results = {country: translate.delay(country, to_lang='zh') for country in countries}
    for country, result in results.items():
        try:
            countries[country]['display']['zh'] = result.get()
        except:
            logger.error("Error during translating %s." % country)
    stats = mongodb().country_stats
    for country in countries:
        stats.update({'_id': country}, {'$set': countries[country]}, True)

@w.task(time_limit=3600*12)
@concurrency(1)
def city_rank():
    '''Activities per city and month.'''
    localities = {}
    default = lambda: {'year': defaultdict(int), 'month': defaultdict(lambda: defaultdict(int)), 'total': 0}
    for user in mongodb().users_stats.find({'loc.city': {'$ne': None}},
                                           {'month': 1, 'loc': 1, 'contrib': 1}):
        country, state, city = [user['loc'].get(t, None) for t in ['country', 'state', 'city']]
        if city not in localities:
            localities[city] = default()
            localities[city]['users'] = 0
            localities[city]['contrib'] = defaultdict(default)
            localities[city]['country'] = country
            localities[city]['state'] = state
            localities[city]['display'] = {'en': city}
        localities[city]['users'] += 1
        for year in user.get('month', {}):
            for month in user['month'][year]:
                localities[city]['month'][year][month] += user['month'][year][month]
                localities[city]['year'][year] += user['month'][year][month]
                localities[city]['total'] += user['month'][year][month]
        cont = localities[city]['contrib']
        for lang in user.get('contrib', {}):
            for year in user['contrib'][lang]:
                for month in user['contrib'][lang][year]:
                    cont[lang]['month'][year][month] += user['contrib'][lang][year][month]
                    cont[lang]['year'][year] += user['contrib'][lang][year][month]
                    cont[lang]['total'] += user['contrib'][lang][year][month]
    results = {city: translate.delay(city, to_lang='zh') for city in localities}
    for city, result in results.items():
        try:
            localities[city]['display']['zh'] = result.get()
        except:
            logger.error("Error during translating %s." % city)
    stats = mongodb().city_stats
    for city, value in localities.items():
        stats.update({'_id': city}, {'$set': value}, True)

@w.task
def translate(text, to_lang='zh'):
    translation = mongodb().translation
    t = translation.find_one({'_id': text, to_lang: {'$ne': None}}, {to_lang: 1})
    result = t and t[to_lang]
    if not result:
        result = Translator(to_lang=to_lang, from_lang='en').translate(text.encode('utf8'))
        translation.update({'_id': text}, {'$set': {to_lang: result}}, True)
    return result

@w.task
@concurrency(1)
def update_user_location():
    locs = {}
    for location in mongodb().locations.find({},
                                             {'locality.long_name': 1,
                                              'country.long_name': 1,
                                              'administrative_area_level_1.long_name': 1
                                              }):
        locs[location['_id']] = {
            'country': location.get('country', {}).get('long_name', None),
            'state': location.get('administrative_area_level_1', {}).get('long_name', None),
            'city': location.get('locality', {}).get('long_name', None)
        }
    users_stats = mongodb().users_stats
    for user in users_stats.find({'info.location': {'$ne': None}, 'loc': None}):
        location = user['info']['location'].lower()
        if location in locs:
            users_stats.update({'_id': user['_id']}, {'$set': {'loc': locs[location]}})
    try:
        country_rank.delay().get()
    except:
        logger.error('Error during ranking country.')
    try:
        city_rank.delay().get()
    except:
        logger.error('Error during ranking city.')

# @w.task
# def lang_rank(lang, year=None):
#     year = year or datetime.now().year
#     key = 'contrib.%s.%d' % (lang, year)
#     users = {}
#     for user in mongodb().users_stats.find({key: {'$ne': None}}, {key: 1}):
#         data = user['contrib'][lang]['%d' % year]
#         users[user['_id']] = sum([data[month] for month in data])
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os

runner = os.environ.get('_')
if runner.endswith('nosetests'):
    try:
        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()
    except ImportError:
        pass

import logging
from ConfigParser import NoSectionError
from collections import defaultdict

from tokenserver.assignment import INodeAssignment

from metlog.logging import hook_logger

from mozsvc.config import get_configurator
from mozsvc.plugin import load_and_register, load_from_settings
from mozsvc.secrets import Secrets

logger = logging.getLogger('tokenserver')

def includeme(config):
    try:
        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()
    except ImportError:
        pass

    config.include("cornice")
    config.include("mozsvc")
    config.scan("tokenserver.views")
    settings = config.registry.settings

    # default metlog setup
    if 'metlog.backend' not in settings:
        settings['metlog.backend'] = 'mozsvc.metrics.MetlogPlugin'
        settings['metlog.enabled'] = True
        settings['metlog.sender_class'] = \
                'metlog.senders.StdOutSender'

    metlog_wrapper = load_from_settings('metlog', settings)

    if settings['metlog.enabled']:
        for logger in ('tokenserver', 'mozsvc', 'powerhose'):
            hook_logger(logger, metlog_wrapper.client)

    config.registry['metlog'] = metlog_wrapper.client

    # initializes the assignment backend
    load_and_register("tokenserver", config)

    # initialize the powerhose and browserid backends if they exist
    for section in ("powerhose", "browserid"):
        try:
            load_and_register(section, config)
        except NoSectionError:
            pass

    # load apps and set them up back in the setting
    key = 'tokenserver.applications'
    applications = defaultdict(list)
    for element in settings.get(key, '').split(','):
        element = element.strip()
        if element == '':
            continue
        element = element.split('-')
        if len(element) != 2:
            continue
        app, version = element
        applications[app].append(version)

    settings[key] = applications

    # load the secrets file(s)
    key = 'tokenserver.secrets_file'
    secret_file = settings[key]
    if not isinstance(secret_file, list):
        secret_file = [secret_file]

    files = []
    for line in secret_file:
        secret_file = [file for file in [file.strip() for file in line.split()]
                       if file != '']
        files.extend(secret_file)

    settings[key] = Secrets(files)
    read_endpoints(config)

class LazyDict(dict):
    def __init__(self, callable):
        self.callable = callable
        self._loaded = False

    def __getitem__(self, name):
        if not self._loaded:
            self.callable(self)
            self._loaded = True
        return super(LazyDict, self).__getitem__(name)

def load_endpoints(mapping, config):
    patterns = dict([(key.split('.', 1)[-1], value)
                        for key, value in config.registry.settings.items()
                        if key.startswith('endpoints.')])
    mapping.update(patterns)

    if len(mapping) == 0:
        # otherwise, try to ask the assignment backend the list of
        # endpoints
        backend = config.registry.getUtility(INodeAssignment)
        mapping.update(backend.get_patterns())

def read_endpoints(config):
    """If there is a section "endpoints", load it the format is
    service-version = pattern, and a dict will be built with those.
    """
    def _read(mapping):
        load_endpoints(mapping, config)

    config.registry['endpoints_patterns'] = LazyDict(_read)

def main(global_config, **settings):
    config = get_configurator(global_config, **settings)
    config.include(includeme)
    return config.make_wsgi_app()
<commit_msg>omit none<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os

runner = os.environ.get('_')
if runner is not None and runner.endswith('nosetests'):
    try:
        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()
    except ImportError:
        pass

import logging
from ConfigParser import NoSectionError
from collections import defaultdict

from tokenserver.assignment import INodeAssignment

from metlog.logging import hook_logger

from mozsvc.config import get_configurator
from mozsvc.plugin import load_and_register, load_from_settings
from mozsvc.secrets import Secrets

logger = logging.getLogger('tokenserver')

def includeme(config):
    try:
        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()
    except ImportError:
        pass

    config.include("cornice")
    config.include("mozsvc")
    config.scan("tokenserver.views")
    settings = config.registry.settings

    # default metlog setup
    if 'metlog.backend' not in settings:
        settings['metlog.backend'] = 'mozsvc.metrics.MetlogPlugin'
        settings['metlog.enabled'] = True
        settings['metlog.sender_class'] = \
                'metlog.senders.StdOutSender'

    metlog_wrapper = load_from_settings('metlog', settings)

    if settings['metlog.enabled']:
        for logger in ('tokenserver', 'mozsvc', 'powerhose'):
            hook_logger(logger, metlog_wrapper.client)

    config.registry['metlog'] = metlog_wrapper.client

    # initializes the assignment backend
    load_and_register("tokenserver", config)

    # initialize the powerhose and browserid backends if they exist
    for section in ("powerhose", "browserid"):
        try:
            load_and_register(section, config)
        except NoSectionError:
            pass

    # load apps and set them up back in the setting
    key = 'tokenserver.applications'
    applications = defaultdict(list)
    for element in settings.get(key, '').split(','):
        element = element.strip()
        if element == '':
            continue
        element = element.split('-')
        if len(element) != 2:
            continue
        app, version = element
        applications[app].append(version)

    settings[key] = applications

    # load the secrets file(s)
    key = 'tokenserver.secrets_file'
    secret_file = settings[key]
    if not isinstance(secret_file, list):
        secret_file = [secret_file]

    files = []
    for line in secret_file:
        secret_file = [file for file in [file.strip() for file in line.split()]
                       if file != '']
        files.extend(secret_file)

    settings[key] = Secrets(files)
    read_endpoints(config)

class LazyDict(dict):
    def __init__(self, callable):
        self.callable = callable
        self._loaded = False

    def __getitem__(self, name):
        if not self._loaded:
            self.callable(self)
            self._loaded = True
        return super(LazyDict, self).__getitem__(name)

def load_endpoints(mapping, config):
    patterns = dict([(key.split('.', 1)[-1], value)
                        for key, value in config.registry.settings.items()
                        if key.startswith('endpoints.')])
    mapping.update(patterns)

    if len(mapping) == 0:
        # otherwise, try to ask the assignment backend the list of
        # endpoints
        backend = config.registry.getUtility(INodeAssignment)
        mapping.update(backend.get_patterns())

def read_endpoints(config):
    """If there is a section "endpoints", load it the format is
    service-version = pattern, and a dict will be built with those.
    """
    def _read(mapping):
        load_endpoints(mapping, config)

    config.registry['endpoints_patterns'] = LazyDict(_read)

def main(global_config, **settings):
    config = get_configurator(global_config, **settings)
    config.include(includeme)
    return config.make_wsgi_app()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import sys
import logging

from dp_tornado.engine.static_handler import StaticHandler
from dp_tornado.engine.engine import EngineSingleton as dpEngineSingleton

engine = dpEngineSingleton()

class Bootstrap(object):
    @staticmethod
    def init_template(engine_path, application_path, template_name):
        valid = True

        for e in ('config', 'controller', 'model', 'helper'):
            if os.path.isdir(os.path.join(application_path, e)):
                valid = False
                break

        if not valid:
            logging.warning('Default directory structure initialzation skipped. Directory not empty.')
            return False

        else:
            import shutil

            template_path = os.path.join(engine_path, 'engine', 'template', template_name)

            if not os.path.isdir(template_path):
                return False

            for root, dirs, files in os.walk(template_path):
                path = root[len(template_path)+1:]
                app_path = os.path.join(application_path, path)

                if path and os.path.isdir(app_path):
                    continue

                if not os.path.isdir(app_path):
                    os.mkdir(app_path)

                for file in files:
                    src = os.path.join(root, file)
                    dest = os.path.join(app_path, file)

                    if not os.path.isfile(dest):
                        if src.endswith('.pyc'):
                            continue

                        shutil.copy(src, dest)

            return True

    @staticmethod
    def init_args():
        import argparse

        parser = argparse.ArgumentParser()

        parser.add_argument('--app-path', help='App Path')

        parser.add_argument('--scheduler-path', help='Scheduler Path')
        parser.add_argument('--scheduler-timeout', type=int, help='Scheduler Timeout')

        parser.add_argument('-i', '--identifier', help='Identifier')
        parser.add_argument('-p', '--port', type=int, help='Binding port')

        return parser.parse_args()

    @staticmethod
    def init_ini(application_path, ini_file, as_cli=False):
        args = Bootstrap.init_args()

        if as_cli:
            engine.ini.server.set('num_processes', 0)
            engine.ini.server.set('debug', False)

        combined_path = engine.ini.static.get('path', default='combined')
        static_prefix = engine.ini.static.get('prefix', default='/s/')
        static_minify = engine.ini.static.get('minify', default=True)

        if combined_path.find('{server_name}') != -1:
            import socket
            combined_path = combined_path.replace('{server_name}', socket.gethostname())

        if static_prefix[-1] != '/':
            static_prefix = '%s/' % static_prefix

        if combined_path[0] == '/':
            combined_path = combined_path[1:]

        if combined_path[-1] == '/':
            combined_path = combined_path[:-1]

        combined_prefix = '%s%s/' % (static_prefix, combined_path)
        combined_url = combined_path
        combined_path = os.path.join(application_path, 'static', combined_path)

        engine.ini.server.set('application_path', application_path)
        engine.ini.server.set('python', sys.executable)

        if args.port:
            engine.ini.server.set('port', args.port)

        # Identifier
        engine.ini.server.set('identifier', engine.helper.misc.uuid.v1())

        # Setup Options
        engine.ini.server.get('max_worker', default=1)
        engine.ini.server.get('num_processes', default=0)
        engine.ini.server.get('port', default=8080)
        engine.ini.server.get('debug', default=False)
        engine.ini.server.get('gzip', default=True)
        engine.ini.crypto.get('key', default='CR$t0-$CR@T')
        engine.ini.session.get('dsn', default=None)
        engine.ini.session.get('expire_in', default=7200)
        engine.ini.server.get('max_body_size', default=1024*1024*10)

        m17n = engine.ini.server.get('m17n', default='').strip()
        m17n = m17n.split(',') if m17n else ['dummy']
        m17n = [e.strip() for e in m17n]

        engine.ini.server.set('m17n', m17n)

        # Static AWS
        engine.ini.static.get('aws_id')
        engine.ini.static.get('aws_secret')
        engine.ini.static.get('aws_bucket')
        engine.ini.static.get('aws_region')
        engine.ini.static.get('aws_endpoint')

        # Scheduler
        engine.ini.scheduler.get('timezone', default='')
        engine.ini.scheduler.get('mode', default='web')

        exception_delegate = engine.ini.logging.get('exception_delegate', default='') or None
        access_logging = engine.ini.logging.get('access', default=False)
        sql_logging = engine.ini.logging.get('sql', default=False)
        aws_logging = engine.ini.logging.get('aws', default=False)
        http_logging = engine.ini.logging.get('http', default=False)

        if exception_delegate:
            try:
                exception_delegate = eval('engine.%s' % exception_delegate)
                engine.logger.set_delegate_handler(exception_delegate)
            except AttributeError:
                logging.error('The specified exception delegate is invalid.')
                exception_delegate = None

        engine.ini.logging.set('exception_delegate', exception_delegate)

        # Initialize Logging
        logging.basicConfig(
            level=logging.DEBUG if access_logging else logging.WARN,
            format='[%(asctime)s][%(levelname)s] %(message)s')

        # SQLAlchemy logging level
        if sql_logging:
            logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

        # AWS logging level (boto
        if not aws_logging:
            logging.getLogger('boto3').setLevel(logging.WARNING)
            logging.getLogger('botocore').setLevel(logging.WARNING)
            logging.getLogger('nose').setLevel(logging.WARNING)
            logging.getLogger('s3transfer').setLevel(logging.WARNING)

        # Requests logging level
        if not http_logging:
            logging.getLogger('requests').setLevel(logging.WARNING)

        # Disable Pillow log
        logging.getLogger('PIL.PngImagePlugin').setLevel(logging.WARNING)

        return {
            'template_path': os.path.join(application_path, 'view'),
            'static_path': os.path.join(application_path, 'static'),
            'static_handler_class': StaticHandler,
            'static_url_prefix': static_prefix,
            'static_minify': static_minify,
            'static_combined_url': combined_url,
            'combined_static_path': combined_path,
            'combined_static_url_prefix': combined_prefix,
            'compressors': {
                'minifier': None
            },
            'debug': engine.ini.server.debug,
            'gzip': engine.ini.server.gzip,
            'cookie_secret': engine.ini.server.get('cookie_secret', default='default_cookie_secret'),
            'ui_modules': {}
        }
<commit_msg>make default requirements.txt when initialization.<commit_after># -*- coding: utf-8 -*-

import os
import sys
import logging

from dp_tornado.engine.static_handler import StaticHandler
from dp_tornado.engine.engine import EngineSingleton as dpEngineSingleton
from dp_tornado.version import __version__

engine = dpEngineSingleton()

class Bootstrap(object):
    @staticmethod
    def init_template(engine_path, application_path, template_name):
        valid = True

        for e in ('config', 'controller', 'model', 'helper'):
            if os.path.isdir(os.path.join(application_path, e)):
                valid = False
                break

        if not valid:
            logging.warning('Default directory structure initialzation skipped. Directory not empty.')
            return False

        else:
            import shutil

            template_path = os.path.join(engine_path, 'engine', 'template', template_name)

            if not os.path.isdir(template_path):
                return False

            for root, dirs, files in os.walk(template_path):
                path = root[len(template_path)+1:]
                app_path = os.path.join(application_path, path)

                if path and os.path.isdir(app_path):
                    continue

                if not os.path.isdir(app_path):
                    os.mkdir(app_path)

                for file in files:
                    src = os.path.join(root, file)
                    dest = os.path.join(app_path, file)

                    if not os.path.isfile(dest):
                        if src.endswith('.pyc'):
                            continue

                        shutil.copy(src, dest)

            # default requirements.txt file.
            with open(os.path.join(application_path, 'requirements.txt'), 'w') as fp:
                fp.write('dp-tornado==%s' % __version__)

            return True

    @staticmethod
    def init_args():
        import argparse

        parser = argparse.ArgumentParser()

        parser.add_argument('--app-path', help='App Path')

        parser.add_argument('--scheduler-path', help='Scheduler Path')
        parser.add_argument('--scheduler-timeout', type=int, help='Scheduler Timeout')

        parser.add_argument('-i', '--identifier', help='Identifier')
        parser.add_argument('-p', '--port', type=int, help='Binding port')

        return parser.parse_args()

    @staticmethod
    def init_ini(application_path, ini_file, as_cli=False):
        args = Bootstrap.init_args()

        if as_cli:
            engine.ini.server.set('num_processes', 0)
            engine.ini.server.set('debug', False)

        combined_path = engine.ini.static.get('path', default='combined')
        static_prefix = engine.ini.static.get('prefix', default='/s/')
        static_minify = engine.ini.static.get('minify', default=True)

        if combined_path.find('{server_name}') != -1:
            import socket
            combined_path = combined_path.replace('{server_name}', socket.gethostname())

        if static_prefix[-1] != '/':
            static_prefix = '%s/' % static_prefix

        if combined_path[0] == '/':
            combined_path = combined_path[1:]

        if combined_path[-1] == '/':
            combined_path = combined_path[:-1]

        combined_prefix = '%s%s/' % (static_prefix, combined_path)
        combined_url = combined_path
        combined_path = os.path.join(application_path, 'static', combined_path)

        engine.ini.server.set('application_path', application_path)
        engine.ini.server.set('python', sys.executable)

        if args.port:
            engine.ini.server.set('port', args.port)

        # Identifier
        engine.ini.server.set('identifier', engine.helper.misc.uuid.v1())

        # Setup Options
        engine.ini.server.get('max_worker', default=1)
        engine.ini.server.get('num_processes', default=0)
        engine.ini.server.get('port', default=8080)
        engine.ini.server.get('debug', default=False)
        engine.ini.server.get('gzip', default=True)
        engine.ini.crypto.get('key', default='CR$t0-$CR@T')
        engine.ini.session.get('dsn', default=None)
        engine.ini.session.get('expire_in', default=7200)
        engine.ini.server.get('max_body_size', default=1024*1024*10)

        m17n = engine.ini.server.get('m17n', default='').strip()
        m17n = m17n.split(',') if m17n else ['dummy']
        m17n = [e.strip() for e in m17n]

        engine.ini.server.set('m17n', m17n)

        # Static AWS
        engine.ini.static.get('aws_id')
        engine.ini.static.get('aws_secret')
        engine.ini.static.get('aws_bucket')
        engine.ini.static.get('aws_region')
        engine.ini.static.get('aws_endpoint')

        # Scheduler
        engine.ini.scheduler.get('timezone', default='')
        engine.ini.scheduler.get('mode', default='web')

        exception_delegate = engine.ini.logging.get('exception_delegate', default='') or None
        access_logging = engine.ini.logging.get('access', default=False)
        sql_logging = engine.ini.logging.get('sql', default=False)
        aws_logging = engine.ini.logging.get('aws', default=False)
        http_logging = engine.ini.logging.get('http', default=False)

        if exception_delegate:
            try:
                exception_delegate = eval('engine.%s' % exception_delegate)
                engine.logger.set_delegate_handler(exception_delegate)
            except AttributeError:
                logging.error('The specified exception delegate is invalid.')
                exception_delegate = None

        engine.ini.logging.set('exception_delegate', exception_delegate)

        # Initialize Logging
        logging.basicConfig(
            level=logging.DEBUG if access_logging else logging.WARN,
            format='[%(asctime)s][%(levelname)s] %(message)s')

        # SQLAlchemy logging level
        if sql_logging:
            logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

        # AWS logging level (boto
        if not aws_logging:
            logging.getLogger('boto3').setLevel(logging.WARNING)
            logging.getLogger('botocore').setLevel(logging.WARNING)
            logging.getLogger('nose').setLevel(logging.WARNING)
            logging.getLogger('s3transfer').setLevel(logging.WARNING)

        # Requests logging level
        if not http_logging:
            logging.getLogger('requests').setLevel(logging.WARNING)

        # Disable Pillow log
        logging.getLogger('PIL.PngImagePlugin').setLevel(logging.WARNING)

        return {
            'template_path': os.path.join(application_path, 'view'),
            'static_path': os.path.join(application_path, 'static'),
            'static_handler_class': StaticHandler,
            'static_url_prefix': static_prefix,
            'static_minify': static_minify,
            'static_combined_url': combined_url,
            'combined_static_path': combined_path,
            'combined_static_url_prefix': combined_prefix,
            'compressors': {
                'minifier': None
            },
            'debug': engine.ini.server.debug,
            'gzip': engine.ini.server.gzip,
            'cookie_secret': engine.ini.server.get('cookie_secret', default='default_cookie_secret'),
            'ui_modules': {}
        }
<|endoftext|>


<commit_before>import os
import errno
from collections import OrderedDict
try:
    import simplejson as json
except ImportError:
    import json

config_file = '_config.json'

class ConfigSchemaError(Exception):
    """Indicates the config does not conform to the expected types."""
    pass

def content_config(path):
    """Returns the configuration for the specified working directory."""
    return from_file(os.path.join(path, config_file))

def theme_config(path):
    """Returns the configuration for the specified theme directory."""
    return from_file(os.path.join(path, theme_config_file))

def from_file(config_file):
    """Returns the configuration from the specified file."""
    if not os.path.exists(config_file):
        return {}
    with file(config_file, 'r') as f:
        return json.load(f)

def read_config(repo_directory):
    """Returns the configuration from the presentation repository."""
    return read_config_file(os.path.join(repo_directory, config_file))

def read_config_file(path):
    """Returns the configuration from the specified file."""
    try:
        with open(path, 'r') as f:
            return json.load(f, object_pairs_hook=OrderedDict)
    except IOError as ex:
        if ex != errno.ENOENT:
            raise
    return {}

def write_config(repo_directory, config):
    """Writes the specified configuration to the presentation repository."""
    return write_config_file(os.path.join(repo_directory, config_file), config)

def write_config_file(path, config):
    """Writes the specified configuration to the specified file."""
    contents = json.dumps(config, indent=4, separators=(',', ': ')) + '\n'
    try:
        with open(path, 'w') as f:
            f.write(contents)
        return True
    except IOError as ex:
        if ex != errno.ENOENT:
            raise
    return False

def get_value(repo_directory, key, expect_type=None):
    """Gets the value of the specified key in the config file."""
    config = read_config(repo_directory)
    value = config.get(key)
    if expect_type and value is not None and not isinstance(value, expect_type):
        raise ConfigSchemaError('Expected config variable %s to be type %s, got %s'
            % (repr(key), repr(expect_type), repr(type(value))))
    return value

def set_value(repo_directory, key, value, strict=True):
    """Sets the value of a particular key in the config file. This has no effect when setting to the same value."""
    if value is None:
        raise ValueError('Argument "value" must not be None.')

    # Read values and do nothing if not making any changes
    config = read_config(repo_directory)
    old = config.get(key)
    if old == value:
        return old

    # Check schema
    if strict and old is not None and not isinstance(old, type(value)):
        raise ConfigSchemaError('Expected config variable %s to be type %s, got %s'
            % (repr(key), repr(type(value)), repr(type(old))))

    # Set new value and save results
    config[key] = value
    write_config(repo_directory, config)
    return old
<commit_msg>Cleanup config.<commit_after>import os
import errno
from collections import OrderedDict
try:
    import simplejson as json
except ImportError:
    import json

config_file = '_config.json'

class ConfigSchemaError(Exception):
    """Indicates the config does not conform to the expected types."""
    pass

def read_config(repo_directory):
    """Returns the configuration from the presentation repository."""
    return read_config_file(os.path.join(repo_directory, config_file))

def read_config_file(path):
    """Returns the configuration from the specified file."""
    try:
        with open(path, 'r') as f:
            return json.load(f, object_pairs_hook=OrderedDict)
    except IOError as ex:
        if ex != errno.ENOENT:
            raise
    return {}

def write_config(repo_directory, config):
    """Writes the specified configuration to the presentation repository."""
    return write_config_file(os.path.join(repo_directory, config_file), config)

def write_config_file(path, config):
    """Writes the specified configuration to the specified file."""
    contents = json.dumps(config, indent=4, separators=(',', ': ')) + '\n'
    try:
        with open(path, 'w') as f:
            f.write(contents)
        return True
    except IOError as ex:
        if ex != errno.ENOENT:
            raise
    return False

def get_value(repo_directory, key, expect_type=None):
    """Gets the value of the specified key in the config file."""
    config = read_config(repo_directory)
    value = config.get(key)
    if expect_type and value is not None and not isinstance(value, expect_type):
        raise ConfigSchemaError('Expected config variable %s to be type %s, got %s'
            % (repr(key), repr(expect_type), repr(type(value))))
    return value

def set_value(repo_directory, key, value, strict=True):
    """Sets the value of a particular key in the config file. This has no effect when setting to the same value."""
    if value is None:
        raise ValueError('Argument "value" must not be None.')

    # Read values and do nothing if not making any changes
    config = read_config(repo_directory)
    old = config.get(key)
    if old == value:
        return old

    # Check schema
    if strict and old is not None and not isinstance(old, type(value)):
        raise ConfigSchemaError('Expected config variable %s to be type %s, got %s'
            % (repr(key), repr(type(value)), repr(type(old))))

    # Set new value and save results
    config[key] = value
    write_config(repo_directory, config)
    return old
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals, print_function, division

import os
import sublime
import sublime_plugin
import threading
import subprocess
import functools
import os.path
import time

git_root_cache = {}

# Goal is to get: "Packages/Git", allowing for people who rename things
def find_plugin_directory():
    if __file__.startswith('./'):
        # ST2, we get "./git/__init__.py" which is pretty useless since we want the part above that
        # However, os.getcwd() is the plugin directory!
        full = os.getcwd()
    else:
        # In a complete inversion, in ST3 when a plugin is loaded we
        # actually can trust __file__. It'll be something like:
        # /Users/dlynch/Library/Application Support/Sublime Text 3/Packages/Git/git/__init__.py
        full = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
    dirname = os.path.split(full)[-1]
    return "Packages/" + dirname.replace(".sublime-package", "")
PLUGIN_DIRECTORY = find_plugin_directory()

def main_thread(callback, *args, **kwargs):
    # sublime.set_timeout gets used to send things onto the main thread
    # most sublime.[something] calls need to be on the main thread
    sublime.set_timeout(functools.partial(callback, *args, **kwargs), 0)

def open_url(url):
    sublime.active_window().run_command('open_url', {"url": url})

def git_root(directory):
    global git_root_cache

    retval = False
    leaf_dir = directory

    if leaf_dir in git_root_cache and git_root_cache[leaf_dir]['expires'] > time.time():
        return git_root_cache[leaf_dir]['retval']

    while directory:
        if os.path.exists(os.path.join(directory, '.git')):
            retval = directory
            break
        parent = os.path.realpath(os.path.join(directory, os.path.pardir))
        if parent == directory:
            # /.. == /
            retval = False
            break
        directory = parent

    git_root_cache[leaf_dir] = {
        'retval': retval,
        'expires': time.time() + 5
    }

    return retval

# for readability code
def git_root_exist(directory):
    return git_root(directory)

def view_contents(view):
    region = sublime.Region(0, view.size())
    return view.substr(region)

def plugin_file(name):
    return PLUGIN_DIRECTORY + '/' + name

def do_when(conditional, command, *args, **kwargs):
    if conditional():
        return command(*args, **kwargs)
    sublime.set_timeout(functools.partial(do_when, conditional, command, *args, **kwargs), 50)

def goto_xy(view, line, col):
    view.run_command("goto_line", {"line": line})
    for i in range(col):
        view.run_command("move", {"by": "characters", "forward": True})

def _make_text_safeish(text, fallback_encoding, method='decode'):
    # The unicode decode here is because sublime converts to unicode inside
    # insert in such a way that unknown characters will cause errors, which is
    # distinctly non-ideal... and there's no way to tell what's coming out of
    # git in output. So...
    try:
        unitext = getattr(text, method)('utf-8')
    except (UnicodeEncodeError, UnicodeDecodeError):
        unitext = getattr(text, method)(fallback_encoding)
    except AttributeError:
        # strongly implies we're already unicode, but just in case let's cast
        # to string
        unitext = str(text)
    return unitext

def _test_paths_for_executable(paths, test_file):
    for directory in paths:
        file_path = os.path.join(directory, test_file)
        if os.path.exists(file_path) and os.access(file_path, os.X_OK):
            return file_path

def find_binary(cmd):
    # It turns out to be difficult to reliably run git, with varying paths
    # and subprocess environments across different platforms. So. Let's hack
    # this a bit.
    # (Yes, I could fall back on a hardline "set your system path properly"
    # attitude. But that involves a lot more arguing with people.)
    path = os.environ.get('PATH', '').split(os.pathsep)
    if os.name == 'nt':
        cmd = cmd + '.exe'

    path = _test_paths_for_executable(path, cmd)

    if not path:
        # /usr/local/bin:/usr/local/git/bin
        if os.name == 'nt':
            extra_paths = (
                os.path.join(os.environ["ProgramFiles"], "Git", "bin"),
                os.path.join(os.environ["ProgramFiles(x86)"], "Git", "bin"),
            )
        else:
            extra_paths = (
                '/usr/local/bin',
                '/usr/local/git/bin',
            )
        path = _test_paths_for_executable(extra_paths, cmd)
    return path
GIT = find_binary('git')
GITK = find_binary('gitk')

class CommandThread(threading.Thread):
    command_lock = threading.Lock()

    def __init__(self, command, on_done, working_dir="", fallback_encoding="", **kwargs):
        threading.Thread.__init__(self)
        self.command = command
        self.on_done = on_done
        self.working_dir = working_dir
        if "stdin" in kwargs:
            self.stdin = kwargs["stdin"].encode()
        else:
            self.stdin = None
        if "stdout" in kwargs:
            self.stdout = kwargs["stdout"]
        else:
            self.stdout = subprocess.PIPE
        self.fallback_encoding = fallback_encoding
        self.kwargs = kwargs

    def run(self):
        # Ignore directories that no longer exist
        if not os.path.isdir(self.working_dir):
            return

        self.command_lock.acquire()
        output = ''
        callback = self.on_done
        try:
            if self.working_dir != "":
                os.chdir(self.working_dir)
            # Windows needs startupinfo in order to start process in background
            startupinfo = None
            if os.name == 'nt':
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW

            shell = False
            if sublime.platform() == 'windows':
                shell = True

            env = os.environ.copy()
            if sublime.platform() == 'windows' and 'HOME' not in env:
                env['HOME'] = env['USERPROFILE']

            # universal_newlines seems to break `log` in python3
            proc = subprocess.Popen(self.command,
                stdout=self.stdout, stderr=subprocess.STDOUT,
                stdin=subprocess.PIPE, startupinfo=startupinfo,
                shell=shell, universal_newlines=False,
                env=env)
            output = proc.communicate(self.stdin)[0]
            if not output:
                output = ''
            output = _make_text_safeish(output, self.fallback_encoding)
        except subprocess.CalledProcessError as e:
            output = e.returncode
        except OSError as e:
            callback = sublime.error_message
            if e.errno == 2:
                output = "{cmd} binary could not be found in PATH\n\nConsider using the {cmd_setting}_command setting for the Git plugin\n\nPATH is: {path}".format(cmd=self.command[0], cmd_setting=self.command[0].replace('-', '_'), path=os.environ['PATH'])
            else:
                output = e.strerror
        finally:
            self.command_lock.release()
            main_thread(callback, output, **self.kwargs)

# A base for all commands
class GitCommand(object):
    may_change_files = False

    def run_command(self, command, callback=None, show_status=True,
            filter_empty_args=True, no_save=False, **kwargs):
        if filter_empty_args:
            command = [arg for arg in command if arg]
        if 'working_dir' not in kwargs:
            kwargs['working_dir'] = self.get_working_dir()
        if 'fallback_encoding' not in kwargs and self.active_view() and self.active_view().settings().get('fallback_encoding'):
            kwargs['fallback_encoding'] = self.active_view().settings().get('fallback_encoding').rpartition('(')[2].rpartition(')')[0]

        s = sublime.load_settings("Git.sublime-settings")
        if s.get('save_first') and self.active_view() and self.active_view().is_dirty() and not no_save:
            self.active_view().run_command('save')
        if command[0] == 'git':
            us = sublime.load_settings('Preferences.sublime-settings')
            if s.get('git_command') or us.get('git_binary'):
                command[0] = s.get('git_command') or us.get('git_binary')
            elif GIT:
                command[0] = GIT
        if command[0] == 'gitk' and s.get('gitk_command'):
            if s.get('gitk_command'):
                command[0] = s.get('gitk_command')
            elif GITK:
                command[0] = GITK
        if command[0] == 'git' and command[1] == 'flow' and s.get('git_flow_command'):
            command[0] = s.get('git_flow_command')
            del(command[1])
        if not callback:
            callback = self.generic_done

        thread = CommandThread(command, callback, **kwargs)
        thread.start()

        if show_status:
            message = kwargs.get('status_message', False) or ' '.join(command)
            sublime.status_message(message)

    def generic_done(self, result, **kw):
        if self.may_change_files and self.active_view() and self.active_view().file_name():
            if self.active_view().is_dirty():
                result = "WARNING: Current view is dirty.\n\n"
            else:
                # just asking the current file to be re-opened doesn't do anything
                print("reverting")
                position = self.active_view().viewport_position()
                self.active_view().run_command('revert')
                do_when(lambda: not self.active_view().is_loading(), lambda: self.active_view().set_viewport_position(position, False))
                # self.active_view().show(position)

        view = self.active_view()
        if view and view.settings().get('live_git_annotations'):
            self.view.run_command('git_annotate')

        if not result.strip():
            return
        self.panel(result)

    def _output_to_view(self, output_file, output, clear=False,
            syntax="Packages/Diff/Diff.tmLanguage", **kwargs):
        output_file.set_syntax_file(syntax)
        args = {
            'output': output,
            'clear': clear
        }
        output_file.run_command('git_scratch_output', args)

    def scratch(self, output, title=False, position=None, **kwargs):
        scratch_file = self.get_window().new_file()
        if title:
            scratch_file.set_name(title)
        scratch_file.set_scratch(True)
        self._output_to_view(scratch_file, output, **kwargs)
        scratch_file.set_read_only(True)
        scratch_file.settings().set('word_wrap', False)
        if position:
            sublime.set_timeout(lambda: scratch_file.set_viewport_position(position), 0)
        return scratch_file

    def panel(self, output, **kwargs):
        if not hasattr(self, 'output_view'):
            self.output_view = self.get_window().get_output_panel("git")
        self.output_view.set_read_only(False)
        self._output_to_view(self.output_view, output, clear=True, **kwargs)
        self.output_view.set_read_only(True)
        self.get_window().run_command("show_panel", {"panel": "output.git"})

    def quick_panel(self, *args, **kwargs):
        self.get_window().show_quick_panel(*args, **kwargs)

# A base for all git commands that work with the entire repository
class GitWindowCommand(GitCommand, sublime_plugin.WindowCommand):
    def active_view(self):
        return self.window.active_view()

    def _active_file_name(self):
        view = self.active_view()
        if view and view.file_name() and len(view.file_name()) > 0:
            return view.file_name()

    @property
    def fallback_encoding(self):
        if self.active_view() and self.active_view().settings().get('fallback_encoding'):
            return self.active_view().settings().get('fallback_encoding').rpartition('(')[2].rpartition(')')[0]

    # If there's no active view or the active view is not a file on the
    # filesystem (e.g. a search results view), we can infer the folder
    # that the user intends Git commands to run against when there's only
    # only one.
    def is_enabled(self):
        if self._active_file_name() or len(self.window.folders()) == 1:
            return bool(git_root(self.get_working_dir()))
        return False

    def get_file_name(self):
        return ''

    def get_relative_file_name(self):
        return ''

    # If there is a file in the active view use that file's directory to
    # search for the Git root.  Otherwise, use the only folder that is
    # open.
    def get_working_dir(self):
        file_name = self._active_file_name()
        if file_name:
            return os.path.realpath(os.path.dirname(file_name))
        else:
            try:  # handle case with no open folder
                return self.window.folders()[0]
            except IndexError:
                return ''

    def get_window(self):
        return self.window

# A base for all git commands that work with the file in the active view
class GitTextCommand(GitCommand, sublime_plugin.TextCommand):
    def active_view(self):
        return self.view

    def is_enabled(self):
        # First, is this actually a file on the file system?
        if self.view.file_name() and len(self.view.file_name()) > 0:
            return bool(git_root(self.get_working_dir()))
        return False

    def get_file_name(self):
        return os.path.basename(self.view.file_name())

    def get_relative_file_name(self):
        working_dir = self.get_working_dir()
        file_path = working_dir.replace(git_root(working_dir), '')[1:]
        file_name = os.path.join(file_path, self.get_file_name())
        return file_name.replace('\\', '/')  # windows issues

    def get_working_dir(self):
        return os.path.realpath(os.path.dirname(self.view.file_name()))

    def get_window(self):
        # Fun discovery: if you switch tabs while a command is working,
        # self.view.window() is None. (Admittedly this is a consequence
        # of my deciding to do async command processing... but, hey,
        # got to live with that now.)
        # I did try tracking the window used at the start of the command
        # and using it instead of view.window() later, but that results
        # panels on a non-visible window, which is especially useless in
        # the case of the quick panel.
        # So, this is not necessarily ideal, but it does work.
        return self.view.window() or sublime.active_window()
<commit_msg>Some windows path errors if os.environ doesn't contain all values<commit_after>from __future__ import absolute_import, unicode_literals, print_function, division

import os
import sublime
import sublime_plugin
import threading
import subprocess
import functools
import os.path
import time

git_root_cache = {}

# Goal is to get: "Packages/Git", allowing for people who rename things
def find_plugin_directory():
    if __file__.startswith('./'):
        # ST2, we get "./git/__init__.py" which is pretty useless since we want the part above that
        # However, os.getcwd() is the plugin directory!
        full = os.getcwd()
    else:
        # In a complete inversion, in ST3 when a plugin is loaded we
        # actually can trust __file__. It'll be something like:
        # /Users/dlynch/Library/Application Support/Sublime Text 3/Packages/Git/git/__init__.py
        full = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
    dirname = os.path.split(full)[-1]
    return "Packages/" + dirname.replace(".sublime-package", "")
PLUGIN_DIRECTORY = find_plugin_directory()

def main_thread(callback, *args, **kwargs):
    # sublime.set_timeout gets used to send things onto the main thread
    # most sublime.[something] calls need to be on the main thread
    sublime.set_timeout(functools.partial(callback, *args, **kwargs), 0)

def open_url(url):
    sublime.active_window().run_command('open_url', {"url": url})

def git_root(directory):
    global git_root_cache

    retval = False
    leaf_dir = directory

    if leaf_dir in git_root_cache and git_root_cache[leaf_dir]['expires'] > time.time():
        return git_root_cache[leaf_dir]['retval']

    while directory:
        if os.path.exists(os.path.join(directory, '.git')):
            retval = directory
            break
        parent = os.path.realpath(os.path.join(directory, os.path.pardir))
        if parent == directory:
            # /.. == /
            retval = False
            break
        directory = parent

    git_root_cache[leaf_dir] = {
        'retval': retval,
        'expires': time.time() + 5
    }

    return retval

# for readability code
def git_root_exist(directory):
    return git_root(directory)

def view_contents(view):
    region = sublime.Region(0, view.size())
    return view.substr(region)

def plugin_file(name):
    return PLUGIN_DIRECTORY + '/' + name

def do_when(conditional, command, *args, **kwargs):
    if conditional():
        return command(*args, **kwargs)
    sublime.set_timeout(functools.partial(do_when, conditional, command, *args, **kwargs), 50)

def goto_xy(view, line, col):
    view.run_command("goto_line", {"line": line})
    for i in range(col):
        view.run_command("move", {"by": "characters", "forward": True})

def _make_text_safeish(text, fallback_encoding, method='decode'):
    # The unicode decode here is because sublime converts to unicode inside
    # insert in such a way that unknown characters will cause errors, which is
    # distinctly non-ideal... and there's no way to tell what's coming out of
    # git in output. So...
    try:
        unitext = getattr(text, method)('utf-8')
    except (UnicodeEncodeError, UnicodeDecodeError):
        unitext = getattr(text, method)(fallback_encoding)
    except AttributeError:
        # strongly implies we're already unicode, but just in case let's cast
        # to string
        unitext = str(text)
    return unitext

def _test_paths_for_executable(paths, test_file):
    for directory in paths:
        file_path = os.path.join(directory, test_file)
        if os.path.exists(file_path) and os.access(file_path, os.X_OK):
            return file_path

def find_binary(cmd):
    # It turns out to be difficult to reliably run git, with varying paths
    # and subprocess environments across different platforms. So. Let's hack
    # this a bit.
    # (Yes, I could fall back on a hardline "set your system path properly"
    # attitude. But that involves a lot more arguing with people.)
    path = os.environ.get('PATH', '').split(os.pathsep)
    if os.name == 'nt':
        cmd = cmd + '.exe'

    path = _test_paths_for_executable(path, cmd)

    if not path:
        # /usr/local/bin:/usr/local/git/bin
        if os.name == 'nt':
            extra_paths = (
                os.path.join(os.environ.get("ProgramFiles", ""), "Git", "bin"),
                os.path.join(os.environ.get("ProgramFiles(x86)", ""), "Git", "bin"),
            )
        else:
            extra_paths = (
                '/usr/local/bin',
                '/usr/local/git/bin',
            )
        path = _test_paths_for_executable(extra_paths, cmd)
    return path
GIT = find_binary('git')
GITK = find_binary('gitk')

class CommandThread(threading.Thread):
    command_lock = threading.Lock()

    def __init__(self, command, on_done, working_dir="", fallback_encoding="", **kwargs):
        threading.Thread.__init__(self)
        self.command = command
        self.on_done = on_done
        self.working_dir = working_dir
        if "stdin" in kwargs:
            self.stdin = kwargs["stdin"].encode()
        else:
            self.stdin = None
        if "stdout" in kwargs:
            self.stdout = kwargs["stdout"]
        else:
            self.stdout = subprocess.PIPE
        self.fallback_encoding = fallback_encoding
        self.kwargs = kwargs

    def run(self):
        # Ignore directories that no longer exist
        if not os.path.isdir(self.working_dir):
            return

        self.command_lock.acquire()
        output = ''
        callback = self.on_done
        try:
            if self.working_dir != "":
                os.chdir(self.working_dir)
            # Windows needs startupinfo in order to start process in background
            startupinfo = None
            if os.name == 'nt':
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW

            shell = False
            if sublime.platform() == 'windows':
                shell = True

            env = os.environ.copy()
            if sublime.platform() == 'windows' and 'HOME' not in env:
                env['HOME'] = env['USERPROFILE']

            # universal_newlines seems to break `log` in python3
            proc = subprocess.Popen(self.command,
                stdout=self.stdout, stderr=subprocess.STDOUT,
                stdin=subprocess.PIPE, startupinfo=startupinfo,
                shell=shell, universal_newlines=False,
                env=env)
            output = proc.communicate(self.stdin)[0]
            if not output:
                output = ''
            output = _make_text_safeish(output, self.fallback_encoding)
        except subprocess.CalledProcessError as e:
            output = e.returncode
        except OSError as e:
            callback = sublime.error_message
            if e.errno == 2:
                global _has_warned
                if not _has_warned:
                    _has_warned = True
                    output = "{cmd} binary could not be found in PATH\n\nConsider using the {cmd_setting}_command setting for the Git plugin\n\nPATH is: {path}".format(cmd=self.command[0], cmd_setting=self.command[0].replace('-', '_'), path=os.environ['PATH'])
            else:
                output = e.strerror
        finally:
            self.command_lock.release()
            main_thread(callback, output, **self.kwargs)

# A base for all commands
class GitCommand(object):
    may_change_files = False

    def run_command(self, command, callback=None, show_status=True,
            filter_empty_args=True, no_save=False, **kwargs):
        if filter_empty_args:
            command = [arg for arg in command if arg]
        if 'working_dir' not in kwargs:
            kwargs['working_dir'] = self.get_working_dir()
        if 'fallback_encoding' not in kwargs and self.active_view() and self.active_view().settings().get('fallback_encoding'):
            kwargs['fallback_encoding'] = self.active_view().settings().get('fallback_encoding').rpartition('(')[2].rpartition(')')[0]

        s = sublime.load_settings("Git.sublime-settings")
        if s.get('save_first') and self.active_view() and self.active_view().is_dirty() and not no_save:
            self.active_view().run_command('save')
        if command[0] == 'git':
            us = sublime.load_settings('Preferences.sublime-settings')
            if s.get('git_command') or us.get('git_binary'):
                command[0] = s.get('git_command') or us.get('git_binary')
            elif GIT:
                command[0] = GIT
        if command[0] == 'gitk' and s.get('gitk_command'):
            if s.get('gitk_command'):
                command[0] = s.get('gitk_command')
            elif GITK:
                command[0] = GITK
        if command[0] == 'git' and command[1] == 'flow' and s.get('git_flow_command'):
            command[0] = s.get('git_flow_command')
            del(command[1])
        if not callback:
            callback = self.generic_done

        thread = CommandThread(command, callback, **kwargs)
        thread.start()

        if show_status:
            message = kwargs.get('status_message', False) or ' '.join(command)
            sublime.status_message(message)

    def generic_done(self, result, **kw):
        if self.may_change_files and self.active_view() and self.active_view().file_name():
            if self.active_view().is_dirty():
                result = "WARNING: Current view is dirty.\n\n"
            else:
                # just asking the current file to be re-opened doesn't do anything
                print("reverting")
                position = self.active_view().viewport_position()
                self.active_view().run_command('revert')
                do_when(lambda: not self.active_view().is_loading(), lambda: self.active_view().set_viewport_position(position, False))
                # self.active_view().show(position)

        view = self.active_view()
        if view and view.settings().get('live_git_annotations'):
            self.view.run_command('git_annotate')

        if not result.strip():
            return
        self.panel(result)

    def _output_to_view(self, output_file, output, clear=False,
            syntax="Packages/Diff/Diff.tmLanguage", **kwargs):
        output_file.set_syntax_file(syntax)
        args = {
            'output': output,
            'clear': clear
        }
        output_file.run_command('git_scratch_output', args)

    def scratch(self, output, title=False, position=None, **kwargs):
        scratch_file = self.get_window().new_file()
        if title:
            scratch_file.set_name(title)
        scratch_file.set_scratch(True)
        self._output_to_view(scratch_file, output, **kwargs)
        scratch_file.set_read_only(True)
        scratch_file.settings().set('word_wrap', False)
        if position:
            sublime.set_timeout(lambda: scratch_file.set_viewport_position(position), 0)
        return scratch_file

    def panel(self, output, **kwargs):
        if not hasattr(self, 'output_view'):
            self.output_view = self.get_window().get_output_panel("git")
        self.output_view.set_read_only(False)
        self._output_to_view(self.output_view, output, clear=True, **kwargs)
        self.output_view.set_read_only(True)
        self.get_window().run_command("show_panel", {"panel": "output.git"})

    def quick_panel(self, *args, **kwargs):
        self.get_window().show_quick_panel(*args, **kwargs)

# A base for all git commands that work with the entire repository
class GitWindowCommand(GitCommand, sublime_plugin.WindowCommand):
    def active_view(self):
        return self.window.active_view()

    def _active_file_name(self):
        view = self.active_view()
        if view and view.file_name() and len(view.file_name()) > 0:
            return view.file_name()

    @property
    def fallback_encoding(self):
        if self.active_view() and self.active_view().settings().get('fallback_encoding'):
            return self.active_view().settings().get('fallback_encoding').rpartition('(')[2].rpartition(')')[0]

    # If there's no active view or the active view is not a file on the
    # filesystem (e.g. a search results view), we can infer the folder
    # that the user intends Git commands to run against when there's only
    # only one.
    def is_enabled(self):
        if self._active_file_name() or len(self.window.folders()) == 1:
            return bool(git_root(self.get_working_dir()))
        return False

    def get_file_name(self):
        return ''

    def get_relative_file_name(self):
        return ''

    # If there is a file in the active view use that file's directory to
    # search for the Git root.  Otherwise, use the only folder that is
    # open.
    def get_working_dir(self):
        file_name = self._active_file_name()
        if file_name:
            return os.path.realpath(os.path.dirname(file_name))
        else:
            try:  # handle case with no open folder
                return self.window.folders()[0]
            except IndexError:
                return ''

    def get_window(self):
        return self.window

# A base for all git commands that work with the file in the active view
class GitTextCommand(GitCommand, sublime_plugin.TextCommand):
    def active_view(self):
        return self.view

    def is_enabled(self):
        # First, is this actually a file on the file system?
        if self.view.file_name() and len(self.view.file_name()) > 0:
            return bool(git_root(self.get_working_dir()))
        return False

    def get_file_name(self):
        return os.path.basename(self.view.file_name())

    def get_relative_file_name(self):
        working_dir = self.get_working_dir()
        file_path = working_dir.replace(git_root(working_dir), '')[1:]
        file_name = os.path.join(file_path, self.get_file_name())
        return file_name.replace('\\', '/')  # windows issues

    def get_working_dir(self):
        return os.path.realpath(os.path.dirname(self.view.file_name()))

    def get_window(self):
        # Fun discovery: if you switch tabs while a command is working,
        # self.view.window() is None. (Admittedly this is a consequence
        # of my deciding to do async command processing... but, hey,
        # got to live with that now.)
        # I did try tracking the window used at the start of the command
        # and using it instead of view.window() later, but that results
        # panels on a non-visible window, which is especially useless in
        # the case of the quick panel.
        # So, this is not necessarily ideal, but it does work.
        return self.view.window() or sublime.active_window()
<|endoftext|>


<commit_before>import datetime
import locale
import os
from decimal import Decimal

try:  # XXX This can't be right.
    locale.setlocale(locale.LC_ALL, "en_US.utf8")
except locale.Error:
    locale.setlocale(locale.LC_ALL, "en_US.UTF-8")

BIRTHDAY = datetime.date(2012, 6, 1)
CARDINALS = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine']
def age():
    today = datetime.date.today()
    nmonths = today.month - BIRTHDAY.month
    plural = 's' if nmonths != 1 else ''
    if nmonths < 10:
        nmonths = CARDINALS[nmonths]
    else:
        nmonths = str(nmonths)
    return "%s month%s" % (nmonths, plural)

db = None # This global is wired in wireup. It's an instance of
          # gittip.postgres.PostgresManager.

# Not sure we won't want this for something yet. Prune if you don't find it in
# the codebase in a month.
OLD_OLD_AMOUNTS= [Decimal(a) for a in ('0.00', '0.08', '0.16', '0.32', '0.64', '1.28')]
OLD_AMOUNTS= [Decimal(a) for a in ('0.25',)]

AMOUNTS= [Decimal(a) for a in ('0.00', '1.00', '3.00', '6.00', '12.00', '24.00')]

__version__ = "~~VERSION~~"

def get_tip(tipper, tippee):
    """Given two user ids, return a Decimal.
    """
    TIP = """\

        SELECT amount
          FROM tips
         WHERE tipper=%s
           AND tippee=%s
      ORDER BY mtime DESC
         LIMIT 1

    """
    rec = db.fetchone(TIP, (tipper, tippee))
    if rec is None:
        tip = Decimal(0.00)
    else:
        tip = rec['amount']
    return tip

def get_backed_amount(participant_id):
    """Given a unicode, return a Decimal.
    """

    BACKED = """\

        SELECT sum(amount) AS backed
          FROM ( SELECT DISTINCT ON (tipper)
                        amount
                      , tipper
                   FROM tips
                   JOIN participants p ON p.id = tipper
                  WHERE tippee=%s
                    AND last_bill_result = ''
               ORDER BY tipper
                      , mtime DESC
                ) AS foo

    """
    rec = db.fetchone(BACKED, (participant_id,))
    if rec is None:
        amount = None
    else:
        amount = rec['backed']  # might be None

    if amount is None:
        amount = Decimal(0.00)

    return amount

def get_number_of_backers(participant_id):
    """Given a unicode, return an int.
    """

    BACKED = """\

        SELECT count(amount) AS nbackers
          FROM ( SELECT DISTINCT ON (tipper)
                        amount
                      , tipper
                   FROM tips
                   JOIN participants p ON p.id = tipper
                  WHERE tippee=%s
                    AND last_bill_result = ''
               ORDER BY tipper
                      , mtime DESC
                ) AS foo
         WHERE amount > 0

    """
    rec = db.fetchone(BACKED, (participant_id,))
    if rec is None:
        nbackers = None
    else:
        nbackers = rec['nbackers']  # might be None

    if nbackers is None:
        nbackers = 0

    return nbackers

def get_tips_and_total(tipper, for_payday=False, db=None):
    """Given a participant id and a date, return a list and a Decimal.

    This function is used to populate a participant's page for their own
    viewing pleasure, and also by the payday function. If for_payday is not
    False it must be a date object.

    A half-injected dependency, that's what db is.

    """
    if db is None:
        from gittip import db

    if for_payday:

        # For payday we want the oldest relationship to be paid first.
        order_by = "ctime ASC"

        # This is where it gets crash-proof.
        # ==================================
        # We need to account for the fact that we may have crashed during
        # Payday and we're re-running that function. We only want to select
        # tips that existed before Payday started, but haven't been processed
        # as part of this Payday yet.
        #
        # It's a bug if the paydays subselect returns > 1 rows.
        #
        # XXX If we crash during Payday and we rerun it after a timezone
        # change, will we get burned? How?

        ts_filter = """\

               AND mtime < %s
               AND ( SELECT id
                       FROM transfers
                      WHERE tipper=t.tipper
                        AND tippee=t.tippee
                        AND timestamp >= %s
                    ) IS NULL

        """
        args = (tipper, for_payday, for_payday)
    else:
        order_by = "amount DESC"
        ts_filter = ""
        args = (tipper,)

    TIPS = """\

        SELECT * FROM (
            SELECT DISTINCT ON (tippee)
                   amount
                 , tippee
                 , t.ctime
                 , p.claimed_time
              FROM tips t
              JOIN participants p ON p.id = t.tippee
             WHERE tipper = %%s
               %s
          ORDER BY tippee
                 , t.mtime DESC
        ) AS foo
        ORDER BY %s
               , tippee

    """ % (ts_filter, order_by)  # XXX, No injections here, right?!
    tips = list(db.fetchall(TIPS, args))

    # Compute the total.
    # ==================
    # For payday we only want to process payments to tippees who have
    # themselves opted into Gittip. For the tipper's profile page we want to
    # show the total amount they've pledged (so they're not surprised when
    # someone *does* start accepting tips and all of a sudden they're hit with
    # bigger charges.

    if for_payday:
        to_total = [t for t in tips if t['claimed_time'] is not None]
    else:
        to_total = tips
    total = sum([t['amount'] for t in to_total])

    if not total:  # XXX Why is this necessary?
        total = Decimal('0.00')

    return tips, total

# canonizer
# =========
# This is an Aspen hook to ensure that requests are served on a certain root
# URL, even if multiple domains point to the application.

class X: pass
canonical_scheme = None
canonical_host = None

def canonize(request):
    """Enforce a certain scheme and hostname. Store these on request as well.
    """
    scheme = request.headers.get('X-Forwarded-Proto', 'http') # per Heroku
    host = request.headers['Host']
    bad_scheme = scheme != canonical_scheme
    bad_host = bool(canonical_host) and (host != canonical_host)
                # '' and False => ''
    if bad_scheme or bad_host:
        url = '%s://%s' % (canonical_scheme, canonical_host)
        if request.line.method in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
            # Redirect to a particular path for idempotent methods.
            url += request.line.uri.path.raw
            if request.line.uri.querystring:
                url += '?' + request.line.uri.querystring.raw
        else:
            # For non-idempotent methods, redirect to homepage.
            url += '/'
        request.redirect(url, permanent=True)

def configure_payments(request):
    # Work-around for https://github.com/balanced/balanced-python/issues/5
    import balanced
    balanced.configure(os.environ['BALANCED_API_SECRET'])
<commit_msg>Setting version to 4.1.14 in gittip/__init__.py.<commit_after>import datetime
import locale
import os
from decimal import Decimal

try:  # XXX This can't be right.
    locale.setlocale(locale.LC_ALL, "en_US.utf8")
except locale.Error:
    locale.setlocale(locale.LC_ALL, "en_US.UTF-8")

BIRTHDAY = datetime.date(2012, 6, 1)
CARDINALS = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine']
def age():
    today = datetime.date.today()
    nmonths = today.month - BIRTHDAY.month
    plural = 's' if nmonths != 1 else ''
    if nmonths < 10:
        nmonths = CARDINALS[nmonths]
    else:
        nmonths = str(nmonths)
    return "%s month%s" % (nmonths, plural)

db = None # This global is wired in wireup. It's an instance of
          # gittip.postgres.PostgresManager.

# Not sure we won't want this for something yet. Prune if you don't find it in
# the codebase in a month.
OLD_OLD_AMOUNTS= [Decimal(a) for a in ('0.00', '0.08', '0.16', '0.32', '0.64', '1.28')]
OLD_AMOUNTS= [Decimal(a) for a in ('0.25',)]

AMOUNTS= [Decimal(a) for a in ('0.00', '1.00', '3.00', '6.00', '12.00', '24.00')]

__version__ = "4.1.14"

def get_tip(tipper, tippee):
    """Given two user ids, return a Decimal.
    """
    TIP = """\

        SELECT amount
          FROM tips
         WHERE tipper=%s
           AND tippee=%s
      ORDER BY mtime DESC
         LIMIT 1

    """
    rec = db.fetchone(TIP, (tipper, tippee))
    if rec is None:
        tip = Decimal(0.00)
    else:
        tip = rec['amount']
    return tip

def get_backed_amount(participant_id):
    """Given a unicode, return a Decimal.
    """

    BACKED = """\

        SELECT sum(amount) AS backed
          FROM ( SELECT DISTINCT ON (tipper)
                        amount
                      , tipper
                   FROM tips
                   JOIN participants p ON p.id = tipper
                  WHERE tippee=%s
                    AND last_bill_result = ''
               ORDER BY tipper
                      , mtime DESC
                ) AS foo

    """
    rec = db.fetchone(BACKED, (participant_id,))
    if rec is None:
        amount = None
    else:
        amount = rec['backed']  # might be None

    if amount is None:
        amount = Decimal(0.00)

    return amount

def get_number_of_backers(participant_id):
    """Given a unicode, return an int.
    """

    BACKED = """\

        SELECT count(amount) AS nbackers
          FROM ( SELECT DISTINCT ON (tipper)
                        amount
                      , tipper
                   FROM tips
                   JOIN participants p ON p.id = tipper
                  WHERE tippee=%s
                    AND last_bill_result = ''
               ORDER BY tipper
                      , mtime DESC
                ) AS foo
         WHERE amount > 0

    """
    rec = db.fetchone(BACKED, (participant_id,))
    if rec is None:
        nbackers = None
    else:
        nbackers = rec['nbackers']  # might be None

    if nbackers is None:
        nbackers = 0

    return nbackers

def get_tips_and_total(tipper, for_payday=False, db=None):
    """Given a participant id and a date, return a list and a Decimal.

    This function is used to populate a participant's page for their own
    viewing pleasure, and also by the payday function. If for_payday is not
    False it must be a date object.

    A half-injected dependency, that's what db is.

    """
    if db is None:
        from gittip import db

    if for_payday:

        # For payday we want the oldest relationship to be paid first.
        order_by = "ctime ASC"

        # This is where it gets crash-proof.
        # ==================================
        # We need to account for the fact that we may have crashed during
        # Payday and we're re-running that function. We only want to select
        # tips that existed before Payday started, but haven't been processed
        # as part of this Payday yet.
        #
        # It's a bug if the paydays subselect returns > 1 rows.
        #
        # XXX If we crash during Payday and we rerun it after a timezone
        # change, will we get burned? How?

        ts_filter = """\

               AND mtime < %s
               AND ( SELECT id
                       FROM transfers
                      WHERE tipper=t.tipper
                        AND tippee=t.tippee
                        AND timestamp >= %s
                    ) IS NULL

        """
        args = (tipper, for_payday, for_payday)
    else:
        order_by = "amount DESC"
        ts_filter = ""
        args = (tipper,)

    TIPS = """\

        SELECT * FROM (
            SELECT DISTINCT ON (tippee)
                   amount
                 , tippee
                 , t.ctime
                 , p.claimed_time
              FROM tips t
              JOIN participants p ON p.id = t.tippee
             WHERE tipper = %%s
               %s
          ORDER BY tippee
                 , t.mtime DESC
        ) AS foo
        ORDER BY %s
               , tippee

    """ % (ts_filter, order_by)  # XXX, No injections here, right?!
    tips = list(db.fetchall(TIPS, args))

    # Compute the total.
    # ==================
    # For payday we only want to process payments to tippees who have
    # themselves opted into Gittip. For the tipper's profile page we want to
    # show the total amount they've pledged (so they're not surprised when
    # someone *does* start accepting tips and all of a sudden they're hit with
    # bigger charges.

    if for_payday:
        to_total = [t for t in tips if t['claimed_time'] is not None]
    else:
        to_total = tips
    total = sum([t['amount'] for t in to_total])

    if not total:  # XXX Why is this necessary?
        total = Decimal('0.00')

    return tips, total

# canonizer
# =========
# This is an Aspen hook to ensure that requests are served on a certain root
# URL, even if multiple domains point to the application.

class X: pass
canonical_scheme = None
canonical_host = None

def canonize(request):
    """Enforce a certain scheme and hostname. Store these on request as well.
    """
    scheme = request.headers.get('X-Forwarded-Proto', 'http') # per Heroku
    host = request.headers['Host']
    bad_scheme = scheme != canonical_scheme
    bad_host = bool(canonical_host) and (host != canonical_host)
                # '' and False => ''
    if bad_scheme or bad_host:
        url = '%s://%s' % (canonical_scheme, canonical_host)
        if request.line.method in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
            # Redirect to a particular path for idempotent methods.
            url += request.line.uri.path.raw
            if request.line.uri.querystring:
                url += '?' + request.line.uri.querystring.raw
        else:
            # For non-idempotent methods, redirect to homepage.
            url += '/'
        request.redirect(url, permanent=True)

def configure_payments(request):
    # Work-around for https://github.com/balanced/balanced-python/issues/5
    import balanced
    balanced.configure(os.environ['BALANCED_API_SECRET'])
<|endoftext|>