<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# moehime / saimoe / filters - basic filter
#
# Copyright (C) 2012 Wang Xuerui <idontknw.wang@gmail.com>
#
# This file is part of moehime.
#
# moehime is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# moehime is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with moehime.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals, division

import re
from functools import partial

from . import filter_manager
from .base import BaseFilter
from ..votecode import VoteCode

_ = lambda x: x

# 匹配角色名/alias，之后对内容进行精确匹配
# Character name/alias, the captured string is then matched
# in an exact mannner
RE_CHARA = re.compile(r'<<(.*?)>>')

# reduce lookups 减少查找
VOTECODE_RECOGNIZE = VoteCode.recognize

def _chara_match_helper(charas, match):
    # record the chara and strip it out of text
    # this way we get pure moe-mon
    # 记录角色并将其从正文中剔除
    # 这样我们就得到纯粹的萌文
    charas.append(match.group(1))
    return ''

def extract_moebun(text):
    charas = []

    helper = partial(_chara_match_helper, charas)
    moebun = RE_CHARA.sub(helper, text).strip()

    return charas, moebun

def posttime_validator_factory(valid_date):
    Y, m, d = valid_date.year, valid_date.month, valid_date.day

    def __is_posttime_valid(date):
        d_Y, d_m, d_d = date.year, date.month, date.day
        d_H, d_i = date.hour, date.minute

        if d_Y != Y or d_m != m or d_d != d:
            return False

        if not 1 <= d_H <= 23:
            return False

        if d_H == 23 and d_m >= 1:
            return False

        return True
    return __is_posttime_valid

@filter_manager.register
class InitialFilter(BaseFilter):
    '''初始过滤器。

    作用是初始化一些投票对象内部的属性，比如提取 code 和角色名之类。

    为了过滤机制正常工作，此过滤器必须存在于过滤序列中，且必须第一个执行。

    '''

    FILTER_NAME = 'initial'

    def _do_judge(self, datum):
        text = datum.full_text
        datum.code, text_w_o_code = VOTECODE_RECOGNIZE(text)

        charas, moebun = extract_moebun(text_w_o_code)
        datum.charas.update(charas)
        datum.text = moebun

        return True, None

@filter_manager.register
class BasicFilter(BaseFilter):
    '''基础过滤器。

    此过滤器对投票执行最基本的判断：发帖时间检查、code 有效性检查、
    萌文不为空检查、角色不为空检查。

    一般情况下请选中此过滤器，并紧随初始过滤器之后。

    '''

    FILTER_NAME = 'basic'

    def __init__(self, config):
        valid_date = self.valid_date = config['date']
        self.date_validator = posttime_validator_factory(valid_date)

    def _do_judge(self, datum):
        if datum.code is None:
            return False, _('正文无 code')

        if len(datum.text) == 0:
            return False, _('除去 code 与角色名的萌文为空')

        if len(datum.charas) == 0:
            return False, _('未识别出任何角色')

        if not self.date_validator(datum.ctime):
            return False, _('投票不在有效时间段')

        return True, None

# vim:ai:et:ts=4:sw=4:sts=4:ff=unix:fenc=utf-8:
<commit_msg>fix bad japanese pronunciation<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# moehime / saimoe / filters - basic filter
#
# Copyright (C) 2012 Wang Xuerui <idontknw.wang@gmail.com>
#
# This file is part of moehime.
#
# moehime is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# moehime is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with moehime.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals, division

import re
from functools import partial

from . import filter_manager
from .base import BaseFilter
from ..votecode import VoteCode

_ = lambda x: x

# 匹配角色名/alias，之后对内容进行精确匹配
# Character name/alias, the captured string is then matched
# in an exact mannner
RE_CHARA = re.compile(r'<<(.*?)>>')

# reduce lookups 减少查找
VOTECODE_RECOGNIZE = VoteCode.recognize

def _chara_match_helper(charas, match):
    # record the chara and strip it out of text
    # this way we get pure moebun
    # 记录角色并将其从正文中剔除
    # 这样我们就得到纯粹的萌文
    charas.append(match.group(1))
    return ''

def extract_moebun(text):
    charas = []

    helper = partial(_chara_match_helper, charas)
    moebun = RE_CHARA.sub(helper, text).strip()

    return charas, moebun

def posttime_validator_factory(valid_date):
    Y, m, d = valid_date.year, valid_date.month, valid_date.day

    def __is_posttime_valid(date):
        d_Y, d_m, d_d = date.year, date.month, date.day
        d_H, d_i = date.hour, date.minute

        if d_Y != Y or d_m != m or d_d != d:
            return False

        if not 1 <= d_H <= 23:
            return False

        if d_H == 23 and d_m >= 1:
            return False

        return True
    return __is_posttime_valid

@filter_manager.register
class InitialFilter(BaseFilter):
    '''初始过滤器。

    作用是初始化一些投票对象内部的属性，比如提取 code 和角色名之类。

    为了过滤机制正常工作，此过滤器必须存在于过滤序列中，且必须第一个执行。

    '''

    FILTER_NAME = 'initial'

    def _do_judge(self, datum):
        text = datum.full_text
        datum.code, text_w_o_code = VOTECODE_RECOGNIZE(text)

        charas, moebun = extract_moebun(text_w_o_code)
        datum.charas.update(charas)
        datum.text = moebun

        return True, None

@filter_manager.register
class BasicFilter(BaseFilter):
    '''基础过滤器。

    此过滤器对投票执行最基本的判断：发帖时间检查、code 有效性检查、
    萌文不为空检查、角色不为空检查。

    一般情况下请选中此过滤器，并紧随初始过滤器之后。

    '''

    FILTER_NAME = 'basic'

    def __init__(self, config):
        valid_date = self.valid_date = config['date']
        self.date_validator = posttime_validator_factory(valid_date)

    def _do_judge(self, datum):
        if datum.code is None:
            return False, _('正文无 code')

        if len(datum.text) == 0:
            return False, _('除去 code 与角色名的萌文为空')

        if len(datum.charas) == 0:
            return False, _('未识别出任何角色')

        if not self.date_validator(datum.ctime):
            return False, _('投票不在有效时间段')

        return True, None

# vim:ai:et:ts=4:sw=4:sts=4:ff=unix:fenc=utf-8:
<|endoftext|>


<commit_before># This file is part of Moksha.
#
# Moksha is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Moksha is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Moksha.  If not, see <http://www.gnu.org/licenses/>.
#
# Copyright 2008, Red Hat, Inc.
# Authors: Luke Macken <lmacken@redhat.com>

import os
import moksha
import logging
import pkg_resources

from tg.controllers import WSGIAppController
from webob import Request
from shove import Shove
from pylons import config
from inspect import isclass
from pylons.i18n import ugettext
from paste.deploy import appconfig
from sqlalchemy import create_engine
from feedcache.cache import Cache

from moksha.exc import MokshaException
from moksha.lib.helpers import (defaultdict, get_moksha_config_path,
                                get_main_app_config_path)
from moksha.wsgiapp import MokshaAppDispatcher

log = logging.getLogger(__name__)

class MokshaMiddleware(object):
    """
    A layer of WSGI middleware that is responsible for setting up the moksha
    environment, as well as handling every request/response in the application.

    If a request for an application comes in (/apps/$NAME), it will dispatch to
    the RootController of that application as defined in it's egg-info.

    """
    def __init__(self, application):
        log.info('Creating Moksha Middleware')
        self.application = application
        self.mokshaapp = MokshaAppDispatcher()

        moksha.apps = {}        # {'app name': tg.TGController/tg.WSGIAppController}
        moksha._widgets = {}    # {'widget name': tw.api.Widget}
        moksha.menus = {}       # {'menu name': moksha.api.menus.MokshaMenu}
        self.engines = {}       # {'app name': sqlalchemy.engine.base.Engine}

        self.load_paths()
        self.load_renderers()
        self.load_configs()
        self.load_widgets()
        self.load_applications()
        self.load_wsgi_applications()
        self.load_models()
        self.load_menus()

        moksha.feed_storage = Shove(config['feed_cache'], compress=True)
        moksha.feed_cache = Cache(moksha.feed_storage)

    def __call__(self, environ, start_response):
        self.register_stomp(environ)
        request = Request(environ)
        if request.path.startswith('/appz/') or request.path.startswith('/widget') or \
           request.path.startswith('/docs/'):
            response = request.get_response(self.mokshaapp)
        else:
            response = request.get_response(self.application)
        return response(environ, start_response)

    def register_stomp(self, environ):
        environ['paste.registry'].register(moksha.stomp, {
            'onopen': [],
            'onclose': [],
            'onerror': [],
            'onerrorframe': [],
            'onconnectedframe': [],
            'onmessageframe': defaultdict(list) # {topic: [js_callback,]}
        })

    def load_paths(self):
        """ Load the names and paths of all moksha applications and widgets.

        We must do this before actually loading the widgets or applications, to
        ensure that we parse and load each of their configuration files
        beforehand.
        """
        for app_entry in pkg_resources.iter_entry_points('moksha.application'):
            if app_entry.name in moksha.apps:
                raise MokshaException('Duplicate application name: %s' %
                                      app_entry.name)
            app_path = app_entry.dist.location
            moksha.apps[app_entry.name] = {
                    'name': app_entry.name,
                    'path': app_path,
                    }
        for widget_entry in pkg_resources.iter_entry_points('moksha.widget'):
            if widget_entry.name in moksha._widgets:
                raise MokshaException('Duplicate widget name: %s' %
                                      widget_entry.name)
            widget_path = widget_entry.dist.location
            moksha._widgets[widget_entry.name] = {
                    'name': widget_entry.name,
                    'path': widget_path,
                    }

    def load_applications(self):
        log.info('Loading moksha applications')
        for app_entry in pkg_resources.iter_entry_points('moksha.application'):
            log.info('Loading %s application' % app_entry.name)
            app_class = app_entry.load()
            app_path = app_entry.dist.location
            moksha.apps[app_entry.name].update({
                    'name': getattr(app_class, 'name', app_entry.name),
                    'controller': app_class(),
                    'path': app_path,
                    'model': None,
                    })
            try:
                model = __import__('%s.model' % app_entry.name,
                                   globals(), locals(),
                                   [app_entry.name])
                moksha.apps[app_entry.name]['model'] = model
            except ImportError:
                pass

    def load_wsgi_applications(self):
        log.info('Loading moksha WSGI applications')
        for app_entry in pkg_resources.iter_entry_points('moksha.wsgiapp'):
            log.info('Loading %s WSGI application' % app_entry.name)
            app_class = app_entry.load()
            app_path = app_entry.dist.location
            moksha.apps[app_entry.name] = {
                    'name': getattr(app_class, 'name', app_entry.name),
                    'controller': WSGIAppController(app_class()),
                    'path': app_path,
                    'model': None,
                    }

    def load_widgets(self):
        log.info('Loading moksha widgets')
        for widget_entry in pkg_resources.iter_entry_points('moksha.widget'):
            log.info('Loading %s widget' % widget_entry.name)
            widget_class = widget_entry.load()
            widget_path = widget_entry.dist.location
            if isclass(widget_class):
                widget = widget_class(widget_entry.name)
            else:
                widget = widget_class
            moksha._widgets[widget_entry.name] = {
                    'name': getattr(widget_class, '__name__',
                                    widget_entry.name),
                    'widget': widget,
                    'path': widget_path,
                    }

        moksha._widgets = moksha._widgets

    def load_menus(self):
        log.info('Loading moksha menus')
        for menu_entry in pkg_resources.iter_entry_points('moksha.menu'):
            log.info('Loading %s menu' % menu_entry.name)
            menu_class = menu_entry.load()
            menu_path = menu_entry.dist.location
            moksha.menus[menu_entry.name] = menu_class(menu_entry.name)

    def load_renderers(self):
        """ Load our template renderers with our application paths.

        We are currently overloading TG2's default Mako renderer because
        the default `escape` filter causes our widgets to show up as escaped HTML.

         """
        #template_paths = config['pylons.paths']['templates']
        #moksha_dir = os.path.abspath(__file__ + '/../../../')
        #for app in [{'path': moksha_dir}] + moksha.apps.values():
        #    if app['path'] not in template_paths:
        #        template_paths.append(app['path'])

        #config['pylons.paths']['templates'] = template_paths

        from mako.template import Template
        from mako.lookup import TemplateLookup
        from tg.util import get_dotted_filename

        class DottedTemplateLookup(object):
            """this is an emulation of the Mako template lookup
            that will handle get_template and support dotted names
            in python path notation to support zipped eggs
            """
            def __init__(self, input_encoding, output_encoding,
                         imports, default_filters):
                self.input_encoding = input_encoding
                self.output_encoding = output_encoding
                self.imports = imports
                self.default_filters = default_filters

            def adjust_uri(self, uri, relativeto):
                """this method is used by mako for filesystem based reasons.
                In dotted lookup land we don't adjust uri so se just return
                the value we are given without any change
                """
                if '.' in uri:
                    """we are in the DottedTemplateLookup system so dots in
                    names should be treated as a python path.
                    Since this method is called by template inheritance we must
                    support dotted names also in the inheritance.
                    """
                    result = get_dotted_filename(template_name=uri,
                                                 template_extension='.mak')

                else:
                    """no dot detected, just return plain name
                    """
                    result = uri

                return result

            def get_template(self, template_name):
                """this is the emulated method that must return a template
                instance based on a given template name
                """
                return Template(open(template_name).read(),
                    input_encoding=self.input_encoding,
                    output_encoding=self.output_encoding,
                    default_filters=self.default_filters,
                    imports=self.imports,
                    lookup=self)

        if config.get('use_dotted_templatenames', True):
            # support dotted names by injecting a slightly different template
            # lookup system that will return templates from dotted template
            # notation.
            config['pylons.app_globals'].mako_lookup = DottedTemplateLookup(
                input_encoding='utf-8', output_encoding='utf-8',
                imports=[], default_filters=[])

        else:
            # if no dotted names support was required we will just setup
            # a file system based template lookup mechanism
            config['pylons.app_globals'].mako_lookup = TemplateLookup(
                directories=self.paths['templates'],
                module_directory=self.paths['templates'],
                input_encoding='utf-8', output_encoding='utf-8',
                filesystem_checks=self.auto_reload_templates)

    def load_configs(self):
        """ Load the configuration files for all applications.

        Here we iterate over all applications, loading their configuration
        files and merging their [DEFAULT] configuration into ours.  This
        requires that applications do not have conflicting configuration
        variable names.  To mitigate this, applications should use some basic
        variable namespacing, such as `myapp.myvariable = myvalue`.

        We first make sure to load up Moksha's configuration, for the cases
        where it is being run as WSGI middleware in a different environment.

        """
        moksha_config_path = os.path.dirname(get_moksha_config_path())
        main_app_config_path = os.path.dirname(get_main_app_config_path())
        loaded_configs = []

        for app in [{'path': moksha_config_path}] + moksha.apps.values():
            for configfile in ('production.ini', 'development.ini'):
                confpath = os.path.join(app['path'], configfile)
                if os.path.exists(confpath):
                    conf = appconfig('config:' + confpath)
                    if app.get('name'):
                        moksha.apps[app['name']]['config'] = conf
                    if app['path'] == main_app_config_path or \
                            confpath in loaded_configs:
                        continue
                    log.info('Loading configuration: %s' % confpath)
                    for entry in conf.global_conf:
                        if entry.startswith('_'):
                            continue
                        if entry in config:
                            log.warning('Conflicting variable: %s' % entry)
                            continue
                        else:
                            config[entry] = conf.global_conf[entry]
                            log.debug('Set `%s` in global config' % entry)
                    loaded_configs.append(confpath)
                    break

    def load_models(self):
        """ Setup the SQLAlchemy database models for all moksha applications.

        This method first looks to see if your application has a
        ``sqlalchemy.url`` set in it's configuration file, and will create a
        SQLAlchemy engine with it.  If it does not exist, Moksha will create an
        engine for your application based on the ``app_db`` configuration,
        which defaults to ``sqlite:///$APPNAME.db``.

        It will then bind the engine to your model's
        :class:`sqlalchemy.MetaData`, and initialize all of your tables,
        if they don't already exist.

        """
        for name, app in moksha.apps.items():
            sa_url = app['config'].get('sqlalchemy.url')
            if sa_url:
                if app['config']['__file__'] == get_moksha_config_path():
                    # Moksha's apps don't specify their own SA url
                    self.engines[name] = create_engine(config['app_db'] % name)
                else:
                    # App has specified its own engine url
                    self.engines[name] = create_engine(sa_url)

            # If a `model` module exists in the application, call it's
            # `init_model` method,and bind the engine to it's `metadata`.
            if app.get('model'):
                log.debug('Creating database engine for %s' % app['name'])
                app['model'].init_model(self.engines[name])
                app['model'].metadata.create_all(bind=self.engines[name])
<commit_msg>Minor formatting tweaks<commit_after># This file is part of Moksha.
#
# Moksha is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Moksha is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Moksha.  If not, see <http://www.gnu.org/licenses/>.
#
# Copyright 2008, Red Hat, Inc.
# Authors: Luke Macken <lmacken@redhat.com>

import os
import moksha
import logging
import pkg_resources

from tg.controllers import WSGIAppController
from webob import Request
from shove import Shove
from pylons import config
from inspect import isclass
from pylons.i18n import ugettext
from paste.deploy import appconfig
from sqlalchemy import create_engine
from feedcache.cache import Cache

from moksha.exc import MokshaException
from moksha.lib.helpers import (defaultdict, get_moksha_config_path,
                                get_main_app_config_path)
from moksha.wsgiapp import MokshaAppDispatcher

log = logging.getLogger(__name__)

class MokshaMiddleware(object):
    """
    A layer of WSGI middleware that is responsible for setting up the moksha
    environment, as well as handling every request/response in the application.

    If a request for an application comes in (/apps/$NAME), it will dispatch to
    the RootController of that application as defined in it's egg-info.

    """
    def __init__(self, application):
        log.info('Creating Moksha Middleware')
        self.application = application
        self.mokshaapp = MokshaAppDispatcher()

        moksha.apps = {}        # {'app name': tg.TGController/tg.WSGIAppController}
        moksha._widgets = {}    # {'widget name': tw.api.Widget}
        moksha.menus = {}       # {'menu name': moksha.api.menus.MokshaMenu}
        self.engines = {}       # {'app name': sqlalchemy.engine.base.Engine}

        self.load_paths()
        self.load_renderers()
        self.load_configs()
        self.load_widgets()
        self.load_applications()
        self.load_wsgi_applications()
        self.load_models()
        self.load_menus()

        moksha.feed_storage = Shove(config['feed_cache'], compress=True)
        moksha.feed_cache = Cache(moksha.feed_storage)

    def __call__(self, environ, start_response):
        self.register_stomp(environ)
        request = Request(environ)
        if request.path.startswith('/appz/') or \
           request.path.startswith('/widget') or \
           request.path.startswith('/docs/'):
            response = request.get_response(self.mokshaapp)
        else:
            response = request.get_response(self.application)
        return response(environ, start_response)

    def register_stomp(self, environ):
        environ['paste.registry'].register(moksha.stomp, {
            'onopen': [],
            'onclose': [],
            'onerror': [],
            'onerrorframe': [],
            'onconnectedframe': [],
            'onmessageframe': defaultdict(list) # {topic: [js_callback,]}
        })

    def load_paths(self):
        """ Load the names and paths of all moksha applications and widgets.

        We must do this before actually loading the widgets or applications, to
        ensure that we parse and load each of their configuration files
        beforehand.
        """
        for app_entry in pkg_resources.iter_entry_points('moksha.application'):
            if app_entry.name in moksha.apps:
                raise MokshaException('Duplicate application name: %s' %
                                      app_entry.name)
            app_path = app_entry.dist.location
            moksha.apps[app_entry.name] = {
                    'name': app_entry.name,
                    'path': app_path,
                    }
        for widget_entry in pkg_resources.iter_entry_points('moksha.widget'):
            if widget_entry.name in moksha._widgets:
                raise MokshaException('Duplicate widget name: %s' %
                                      widget_entry.name)
            widget_path = widget_entry.dist.location
            moksha._widgets[widget_entry.name] = {
                    'name': widget_entry.name,
                    'path': widget_path,
                    }

    def load_applications(self):
        log.info('Loading moksha applications')
        for app_entry in pkg_resources.iter_entry_points('moksha.application'):
            log.info('Loading %s application' % app_entry.name)
            app_class = app_entry.load()
            app_path = app_entry.dist.location
            moksha.apps[app_entry.name].update({
                    'name': getattr(app_class, 'name', app_entry.name),
                    'controller': app_class(),
                    'path': app_path,
                    'model': None,
                    })
            try:
                model = __import__('%s.model' % app_entry.name,
                                   globals(), locals(),
                                   [app_entry.name])
                moksha.apps[app_entry.name]['model'] = model
            except ImportError:
                pass

    def load_wsgi_applications(self):
        log.info('Loading moksha WSGI applications')
        for app_entry in pkg_resources.iter_entry_points('moksha.wsgiapp'):
            log.info('Loading %s WSGI application' % app_entry.name)
            app_class = app_entry.load()
            app_path = app_entry.dist.location
            moksha.apps[app_entry.name] = {
                    'name': getattr(app_class, 'name', app_entry.name),
                    'controller': WSGIAppController(app_class()),
                    'path': app_path,
                    'model': None,
                    }

    def load_widgets(self):
        log.info('Loading moksha widgets')
        for widget_entry in pkg_resources.iter_entry_points('moksha.widget'):
            log.info('Loading %s widget' % widget_entry.name)
            widget_class = widget_entry.load()
            widget_path = widget_entry.dist.location
            if isclass(widget_class):
                widget = widget_class(widget_entry.name)
            else:
                widget = widget_class
            moksha._widgets[widget_entry.name] = {
                    'name': getattr(widget_class, '__name__',
                                    widget_entry.name),
                    'widget': widget,
                    'path': widget_path,
                    }

        moksha._widgets = moksha._widgets

    def load_menus(self):
        log.info('Loading moksha menus')
        for menu_entry in pkg_resources.iter_entry_points('moksha.menu'):
            log.info('Loading %s menu' % menu_entry.name)
            menu_class = menu_entry.load()
            menu_path = menu_entry.dist.location
            moksha.menus[menu_entry.name] = menu_class(menu_entry.name)

    def load_renderers(self):
        """ Load our template renderers with our application paths.

        We are currently overloading TG2's default Mako renderer because
        the default `escape` filter causes our widgets to show up as escaped HTML.

         """
        #template_paths = config['pylons.paths']['templates']
        #moksha_dir = os.path.abspath(__file__ + '/../../../')
        #for app in [{'path': moksha_dir}] + moksha.apps.values():
        #    if app['path'] not in template_paths:
        #        template_paths.append(app['path'])

        #config['pylons.paths']['templates'] = template_paths

        from mako.template import Template
        from mako.lookup import TemplateLookup
        from tg.util import get_dotted_filename

        class DottedTemplateLookup(object):
            """this is an emulation of the Mako template lookup
            that will handle get_template and support dotted names
            in python path notation to support zipped eggs
            """
            def __init__(self, input_encoding, output_encoding,
                         imports, default_filters):
                self.input_encoding = input_encoding
                self.output_encoding = output_encoding
                self.imports = imports
                self.default_filters = default_filters

            def adjust_uri(self, uri, relativeto):
                """this method is used by mako for filesystem based reasons.
                In dotted lookup land we don't adjust uri so se just return
                the value we are given without any change
                """
                if '.' in uri:
                    """we are in the DottedTemplateLookup system so dots in
                    names should be treated as a python path.
                    Since this method is called by template inheritance we must
                    support dotted names also in the inheritance.
                    """
                    result = get_dotted_filename(template_name=uri,
                                                 template_extension='.mak')

                else:
                    """no dot detected, just return plain name
                    """
                    result = uri

                return result

            def get_template(self, template_name):
                """this is the emulated method that must return a template
                instance based on a given template name
                """
                return Template(open(template_name).read(),
                    input_encoding=self.input_encoding,
                    output_encoding=self.output_encoding,
                    default_filters=self.default_filters,
                    imports=self.imports,
                    lookup=self)

        if config.get('use_dotted_templatenames', True):
            # support dotted names by injecting a slightly different template
            # lookup system that will return templates from dotted template
            # notation.
            config['pylons.app_globals'].mako_lookup = DottedTemplateLookup(
                input_encoding='utf-8', output_encoding='utf-8',
                imports=[], default_filters=[])

        else:
            # if no dotted names support was required we will just setup
            # a file system based template lookup mechanism
            config['pylons.app_globals'].mako_lookup = TemplateLookup(
                directories=self.paths['templates'],
                module_directory=self.paths['templates'],
                input_encoding='utf-8', output_encoding='utf-8',
                filesystem_checks=self.auto_reload_templates)

    def load_configs(self):
        """ Load the configuration files for all applications.

        Here we iterate over all applications, loading their configuration
        files and merging their [DEFAULT] configuration into ours.  This
        requires that applications do not have conflicting configuration
        variable names.  To mitigate this, applications should use some basic
        variable namespacing, such as `myapp.myvariable = myvalue`.

        We first make sure to load up Moksha's configuration, for the cases
        where it is being run as WSGI middleware in a different environment.

        """
        moksha_config_path = os.path.dirname(get_moksha_config_path())
        main_app_config_path = os.path.dirname(get_main_app_config_path())
        loaded_configs = []

        for app in [{'path': moksha_config_path}] + moksha.apps.values():
            for configfile in ('production.ini', 'development.ini'):
                confpath = os.path.join(app['path'], configfile)
                if os.path.exists(confpath):
                    conf = appconfig('config:' + confpath)
                    if app.get('name'):
                        moksha.apps[app['name']]['config'] = conf
                    if app['path'] == main_app_config_path or \
                            confpath in loaded_configs:
                        continue
                    log.info('Loading configuration: %s' % confpath)
                    for entry in conf.global_conf:
                        if entry.startswith('_'):
                            continue
                        if entry in config:
                            log.warning('Conflicting variable: %s' % entry)
                            continue
                        else:
                            config[entry] = conf.global_conf[entry]
                            log.debug('Set `%s` in global config' % entry)
                    loaded_configs.append(confpath)
                    break

    def load_models(self):
        """ Setup the SQLAlchemy database models for all moksha applications.

        This method first looks to see if your application has a
        ``sqlalchemy.url`` set in it's configuration file, and will create a
        SQLAlchemy engine with it.  If it does not exist, Moksha will create an
        engine for your application based on the ``app_db`` configuration,
        which defaults to ``sqlite:///$APPNAME.db``.

        It will then bind the engine to your model's
        :class:`sqlalchemy.MetaData`, and initialize all of your tables,
        if they don't already exist.

        """
        for name, app in moksha.apps.items():
            sa_url = app['config'].get('sqlalchemy.url')
            if sa_url:
                if app['config']['__file__'] == get_moksha_config_path():
                    # Moksha's apps don't specify their own SA url
                    self.engines[name] = create_engine(config['app_db'] % name)
                else:
                    # App has specified its own engine url
                    self.engines[name] = create_engine(sa_url)

            # If a `model` module exists in the application, call it's
            # `init_model` method,and bind the engine to it's `metadata`.
            if app.get('model'):
                log.debug('Creating database engine for %s' % app['name'])
                app['model'].init_model(self.engines[name])
                app['model'].metadata.create_all(bind=self.engines[name])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
import os, platform, locale, stat
import shutil, re, string, shutil
from optparse import OptionParser
import configparser
import glob
import common.print_color
import common.project_utils as project

console_encoding = sys.getfilesystemencoding()
script_dir = os.path.dirname(os.path.realpath(__file__))

if __name__ == '__main__':
    os.chdir(script_dir)
    from mako.template import Template
    from mako.lookup import TemplateLookup
    etc_template_dir = os.path.join(script_dir, 'helper', 'template', 'etc')
    script_template_dir = os.path.join(script_dir, 'helper', 'template', 'script')
    project_lookup = TemplateLookup(directories=[etc_template_dir, script_template_dir], module_directory = os.path.join(script_dir, '.mako_modules'))

    parser = OptionParser("usage: %prog [options...]")
    parser.add_option("-e", "--env-prefix", action='store', dest="env_prefix", default='AUTOBUILD_', help="prefix when read parameters from environment variables")
    parser.add_option("-c", "--config", action='store', dest="config", default=os.path.join(script_dir, 'config.conf'), help="configure file(default: {0})".format(os.path.join(script_dir, 'config.conf')))
    parser.add_option("-s", "--set", action='append' , dest="set_vars", default=[], help="set configures")
    parser.add_option("-n", "--number", action='store' , dest="reset_number", default=None, type='int', help="set default server numbers")
    parser.add_option("-i", "--id-offset", action='store' , dest="server_id_offset", default=0, type='int', help="set server id offset(default: 0)")

    (opts, args) = parser.parse_args()
    config = configparser.ConfigParser(inline_comment_prefixes=('#', ';'))
    config.read(opts.config)

    project.set_global_opts(config, opts.server_id_offset)

    # all custon environment start with SYSTEM_MACRO_CUSTOM_
    # reset all servers's number
    if opts.reset_number is not None:
        for svr_name in project.get_global_all_services():
            config.set('server.{0}'.format(svr_name), 'number', opts.reset_number)

    # set all custom configures
    ext_cmd_rule = re.compile('(.*)\.([^\.]+)=([^=]*)$')
    for cmd in opts.set_vars:
        mat_res = ext_cmd_rule.match(cmd)
        if mat_res:
            config.set(mat_res.group(1), mat_res.group(2), mat_res.group(3))
        else:
            common.print_color.cprintf_stdout([common.print_color.print_style.FC_RED, common.print_color.print_style.FW_BOLD], 'set command {0} invalid, must be SECTION.KEY=VALUE\r\n', cmd)

    # copy script templates
    restart_all_script = os.path.join(script_dir, 'restart_all.sh')
    reload_all_script = os.path.join(script_dir, 'reload_all.sh')
    stop_all_script = os.path.join(script_dir, 'stop_all.sh')
    shutil.copy2(
        os.path.join(script_dir, 'helper', 'template', 'script', 'restart_all.template.sh'),
        restart_all_script
    )
    os.chmod(restart_all_script, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
    shutil.copy2(
        os.path.join(script_dir, 'helper', 'template', 'script', 'reload_all.template.sh'),
        reload_all_script
    )
    os.chmod(reload_all_script, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
    shutil.copy2(
        os.path.join(script_dir, 'helper', 'template', 'script', 'stop_all.template.sh'),
        stop_all_script
    )
    os.chmod(stop_all_script, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
    restart_all_content = []
    reload_all_content = []
    stop_all_content = []

    def generate_service(svr_name, svr_index, install_prefix, section_name, **ext_options):
        project.set_server_inst(config[section_name], svr_name, svr_index)
        common.print_color.cprintf_stdout([common.print_color.print_style.FC_YELLOW, common.print_color.print_style.FW_BOLD], 'start to generate etc and script of {0}-{1}\r\n', svr_name, svr_index)

        etc_in_name = '{0}.conf'.format(svr_name)
        etc_out_name = '{0}-{1}.conf'.format(svr_name, svr_index)
        start_script_name = 'start-{0}.sh'.format(svr_index)
        stop_script_name = 'stop-{0}.sh'.format(svr_index)
        reload_script_name = 'reload-{0}.sh'.format(svr_index)
        install_abs_prefix = os.path.normpath(os.path.join(script_dir, '..', '..', install_prefix))

        if not os.path.exists(os.path.join(install_abs_prefix, 'etc')):
            os.makedirs(os.path.join(install_abs_prefix, 'etc'))
        if not os.path.exists(os.path.join(install_abs_prefix, 'bin')):
            os.makedirs(os.path.join(install_abs_prefix, 'bin'))
        # etc
        gen_in_path = os.path.join(etc_template_dir, etc_in_name)
        gen_out_path = os.path.join(install_abs_prefix, 'etc', etc_out_name)
        if os.path.exists(gen_in_path):
            svr_tmpl = project_lookup.get_template(etc_in_name)
            open(gen_out_path, mode='w').write(svr_tmpl.render(
                project_install_prefix=os.path.relpath('.', os.path.join(install_prefix, 'etc')),
                **ext_options
            ))
            os.chmod(gen_out_path, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
        # start script
        gen_in_path = os.path.join(script_template_dir, 'start.sh')
        gen_out_path = os.path.join(install_abs_prefix, 'bin', start_script_name)
        if os.path.exists(gen_in_path):
            svr_tmpl = project_lookup.get_template('start.sh')
            open(gen_out_path, mode='w').write(svr_tmpl.render(
                project_install_prefix=os.path.relpath('.', os.path.join(install_prefix, 'bin')),
                **ext_options
            ))
            os.chmod(gen_out_path, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
            restart_all_content.append("""
# ==================== {0} ====================
if [ $# -eq 0 ] || [ "0" == "$(is_in_server_list {1} $*)" ]; then
    bash {2}
fi
            """.format(
                project.get_server_full_name(),
                project.get_server_full_name(),
                os.path.relpath(gen_out_path, script_dir)
            ))

        # stop script
        gen_in_path = os.path.join(script_template_dir, 'stop.sh')
        gen_out_path = os.path.join(install_abs_prefix, 'bin', stop_script_name)
        if os.path.exists(gen_in_path):
            svr_tmpl = project_lookup.get_template('stop.sh')
            open(gen_out_path, mode='w').write(svr_tmpl.render(
                project_install_prefix=os.path.relpath('.', os.path.join(install_prefix, 'bin')),
                **ext_options
            ))
            os.chmod(gen_out_path, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
            stop_all_content.append("""
# ==================== {0} ====================
if [ $# -eq 0 ] || [ "0" == "$(is_in_server_list {1} $*)" ]; then
    bash {2}
fi
            """.format(
                project.get_server_full_name(),
                project.get_server_full_name(),
                os.path.relpath(gen_out_path, script_dir)
            ))

        # reload script
        gen_in_path = os.path.join(script_template_dir, 'reload.sh')
        gen_out_path = os.path.join(install_abs_prefix, 'bin', reload_script_name)
        if os.path.exists(gen_in_path):
            svr_tmpl = project_lookup.get_template('reload.sh')
            open(gen_out_path, mode='w').write(svr_tmpl.render(
                project_install_prefix=os.path.relpath('.', os.path.join(install_prefix, 'bin')),
                **ext_options
            ))
            os.chmod(gen_out_path, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
            reload_all_content.append("""
# ==================== {0} ====================
if [ $# -eq 0 ] || [ "0" == "$(is_in_server_list {1} $*)" ]; then
    bash {2}
fi
            """.format(
                project.get_server_full_name(),
                project.get_server_full_name(),
                os.path.relpath(gen_out_path, script_dir)
            ))

    # parse all services
    atgateway_index = 1 + opts.server_id_offset
    for svr_name in project.get_global_all_services():
        section_name = 'server.{0}'.format(svr_name)
        install_prefix = project.get_global_option(section_name, 'install_prefix', svr_name)
        for svr_index in project.get_service_index_range(int(project.get_global_option(section_name, 'number', 0))):
            generate_service(svr_name, svr_index, install_prefix, section_name)
            # atgateway if available
            atgateway_port = project.get_server_option('atgateway_port', None)
            if not atgateway_port is None:
                atgateway_install_prefix = project.get_global_option('server.atgateway', 'install_prefix', 'atframe/atgateway')
                generate_service('atgateway', atgateway_index, atgateway_install_prefix, section_name,
                    atgateway_server_name = svr_name,
                    atgateway_server_index = svr_index
                )
                atgateway_index = atgateway_index + 1

    open(restart_all_script, mode='a').write(os.linesep.join(restart_all_content))
    open(reload_all_script, mode='a').write(os.linesep.join(reload_all_content))
    stop_all_content.reverse()
    open(stop_all_script, mode='a').write(os.linesep.join(stop_all_content))
    common.print_color.cprintf_stdout([common.print_color.print_style.FC_YELLOW, common.print_color.print_style.FW_BOLD], 'all jobs done.\r\n')
        <commit_msg>use /usr/bin/python3 for finding python3<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
import os, platform, locale, stat
import shutil, re, string, shutil
from optparse import OptionParser
import configparser
import glob
import common.print_color
import common.project_utils as project

console_encoding = sys.getfilesystemencoding()
script_dir = os.path.dirname(os.path.realpath(__file__))

if __name__ == '__main__':
    os.chdir(script_dir)
    from mako.template import Template
    from mako.lookup import TemplateLookup
    etc_template_dir = os.path.join(script_dir, 'helper', 'template', 'etc')
    script_template_dir = os.path.join(script_dir, 'helper', 'template', 'script')
    project_lookup = TemplateLookup(directories=[etc_template_dir, script_template_dir], module_directory = os.path.join(script_dir, '.mako_modules'))

    parser = OptionParser("usage: %prog [options...]")
    parser.add_option("-e", "--env-prefix", action='store', dest="env_prefix", default='AUTOBUILD_', help="prefix when read parameters from environment variables")
    parser.add_option("-c", "--config", action='store', dest="config", default=os.path.join(script_dir, 'config.conf'), help="configure file(default: {0})".format(os.path.join(script_dir, 'config.conf')))
    parser.add_option("-s", "--set", action='append' , dest="set_vars", default=[], help="set configures")
    parser.add_option("-n", "--number", action='store' , dest="reset_number", default=None, type='int', help="set default server numbers")
    parser.add_option("-i", "--id-offset", action='store' , dest="server_id_offset", default=0, type='int', help="set server id offset(default: 0)")

    (opts, args) = parser.parse_args()
    config = configparser.ConfigParser(inline_comment_prefixes=('#', ';'))
    config.read(opts.config)

    project.set_global_opts(config, opts.server_id_offset)

    # all custon environment start with SYSTEM_MACRO_CUSTOM_
    # reset all servers's number
    if opts.reset_number is not None:
        for svr_name in project.get_global_all_services():
            config.set('server.{0}'.format(svr_name), 'number', opts.reset_number)

    # set all custom configures
    ext_cmd_rule = re.compile('(.*)\.([^\.]+)=([^=]*)$')
    for cmd in opts.set_vars:
        mat_res = ext_cmd_rule.match(cmd)
        if mat_res:
            config.set(mat_res.group(1), mat_res.group(2), mat_res.group(3))
        else:
            common.print_color.cprintf_stdout([common.print_color.print_style.FC_RED, common.print_color.print_style.FW_BOLD], 'set command {0} invalid, must be SECTION.KEY=VALUE\r\n', cmd)

    # copy script templates
    restart_all_script = os.path.join(script_dir, 'restart_all.sh')
    reload_all_script = os.path.join(script_dir, 'reload_all.sh')
    stop_all_script = os.path.join(script_dir, 'stop_all.sh')
    shutil.copy2(
        os.path.join(script_dir, 'helper', 'template', 'script', 'restart_all.template.sh'),
        restart_all_script
    )
    os.chmod(restart_all_script, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
    shutil.copy2(
        os.path.join(script_dir, 'helper', 'template', 'script', 'reload_all.template.sh'),
        reload_all_script
    )
    os.chmod(reload_all_script, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
    shutil.copy2(
        os.path.join(script_dir, 'helper', 'template', 'script', 'stop_all.template.sh'),
        stop_all_script
    )
    os.chmod(stop_all_script, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
    restart_all_content = []
    reload_all_content = []
    stop_all_content = []

    def generate_service(svr_name, svr_index, install_prefix, section_name, **ext_options):
        project.set_server_inst(config[section_name], svr_name, svr_index)
        common.print_color.cprintf_stdout([common.print_color.print_style.FC_YELLOW, common.print_color.print_style.FW_BOLD], 'start to generate etc and script of {0}-{1}\r\n', svr_name, svr_index)

        etc_in_name = '{0}.conf'.format(svr_name)
        etc_out_name = '{0}-{1}.conf'.format(svr_name, svr_index)
        start_script_name = 'start-{0}.sh'.format(svr_index)
        stop_script_name = 'stop-{0}.sh'.format(svr_index)
        reload_script_name = 'reload-{0}.sh'.format(svr_index)
        install_abs_prefix = os.path.normpath(os.path.join(script_dir, '..', '..', install_prefix))

        if not os.path.exists(os.path.join(install_abs_prefix, 'etc')):
            os.makedirs(os.path.join(install_abs_prefix, 'etc'))
        if not os.path.exists(os.path.join(install_abs_prefix, 'bin')):
            os.makedirs(os.path.join(install_abs_prefix, 'bin'))
        # etc
        gen_in_path = os.path.join(etc_template_dir, etc_in_name)
        gen_out_path = os.path.join(install_abs_prefix, 'etc', etc_out_name)
        if os.path.exists(gen_in_path):
            svr_tmpl = project_lookup.get_template(etc_in_name)
            open(gen_out_path, mode='w').write(svr_tmpl.render(
                project_install_prefix=os.path.relpath('.', os.path.join(install_prefix, 'etc')),
                **ext_options
            ))
            os.chmod(gen_out_path, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
        # start script
        gen_in_path = os.path.join(script_template_dir, 'start.sh')
        gen_out_path = os.path.join(install_abs_prefix, 'bin', start_script_name)
        if os.path.exists(gen_in_path):
            svr_tmpl = project_lookup.get_template('start.sh')
            open(gen_out_path, mode='w').write(svr_tmpl.render(
                project_install_prefix=os.path.relpath('.', os.path.join(install_prefix, 'bin')),
                **ext_options
            ))
            os.chmod(gen_out_path, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
            restart_all_content.append("""
# ==================== {0} ====================
if [ $# -eq 0 ] || [ "0" == "$(is_in_server_list {1} $*)" ]; then
    bash {2}
fi
            """.format(
                project.get_server_full_name(),
                project.get_server_full_name(),
                os.path.relpath(gen_out_path, script_dir)
            ))

        # stop script
        gen_in_path = os.path.join(script_template_dir, 'stop.sh')
        gen_out_path = os.path.join(install_abs_prefix, 'bin', stop_script_name)
        if os.path.exists(gen_in_path):
            svr_tmpl = project_lookup.get_template('stop.sh')
            open(gen_out_path, mode='w').write(svr_tmpl.render(
                project_install_prefix=os.path.relpath('.', os.path.join(install_prefix, 'bin')),
                **ext_options
            ))
            os.chmod(gen_out_path, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
            stop_all_content.append("""
# ==================== {0} ====================
if [ $# -eq 0 ] || [ "0" == "$(is_in_server_list {1} $*)" ]; then
    bash {2}
fi
            """.format(
                project.get_server_full_name(),
                project.get_server_full_name(),
                os.path.relpath(gen_out_path, script_dir)
            ))

        # reload script
        gen_in_path = os.path.join(script_template_dir, 'reload.sh')
        gen_out_path = os.path.join(install_abs_prefix, 'bin', reload_script_name)
        if os.path.exists(gen_in_path):
            svr_tmpl = project_lookup.get_template('reload.sh')
            open(gen_out_path, mode='w').write(svr_tmpl.render(
                project_install_prefix=os.path.relpath('.', os.path.join(install_prefix, 'bin')),
                **ext_options
            ))
            os.chmod(gen_out_path, stat.S_IRWXU + stat.S_IRWXG + stat.S_IROTH + stat.S_IXOTH)
            reload_all_content.append("""
# ==================== {0} ====================
if [ $# -eq 0 ] || [ "0" == "$(is_in_server_list {1} $*)" ]; then
    bash {2}
fi
            """.format(
                project.get_server_full_name(),
                project.get_server_full_name(),
                os.path.relpath(gen_out_path, script_dir)
            ))

    # parse all services
    atgateway_index = 1 + opts.server_id_offset
    for svr_name in project.get_global_all_services():
        section_name = 'server.{0}'.format(svr_name)
        install_prefix = project.get_global_option(section_name, 'install_prefix', svr_name)
        for svr_index in project.get_service_index_range(int(project.get_global_option(section_name, 'number', 0))):
            generate_service(svr_name, svr_index, install_prefix, section_name)
            # atgateway if available
            atgateway_port = project.get_server_option('atgateway_port', None)
            if not atgateway_port is None:
                atgateway_install_prefix = project.get_global_option('server.atgateway', 'install_prefix', 'atframe/atgateway')
                generate_service('atgateway', atgateway_index, atgateway_install_prefix, section_name,
                    atgateway_server_name = svr_name,
                    atgateway_server_index = svr_index
                )
                atgateway_index = atgateway_index + 1

    open(restart_all_script, mode='a').write(os.linesep.join(restart_all_content))
    open(reload_all_script, mode='a').write(os.linesep.join(reload_all_content))
    stop_all_content.reverse()
    open(stop_all_script, mode='a').write(os.linesep.join(stop_all_content))
    common.print_color.cprintf_stdout([common.print_color.print_style.FC_YELLOW, common.print_color.print_style.FW_BOLD], 'all jobs done.\r\n')
        <|endoftext|>


<commit_before>import sys
from collections import MutableMapping
from types import MethodType

from django.db.models.fields import FieldDoesNotExist
from django.utils.text import capfirst
from django.utils.functional import LazyObject, new_method_proxy
try:
    # New in Django 1.7+
    from django.utils.text import camel_case_to_spaces
except ImportError:
    # Backwards compatibility
    from django.db.models.options import get_verbose_name as camel_case_to_spaces
from django.conf import settings
from mongoengine.fields import ReferenceField, ListField

def patch_document(function, instance, bound=True):
    if bound:
        method = MethodType(function, instance)
    else:
        method = function
    setattr(instance, function.__name__, method)

def create_verbose_name(name):
    name = camel_case_to_spaces(name)
    name = name.replace('_', ' ')
    return name

class Relation(object):
    # just an empty dict to make it useable with Django
    # mongoengine has no notion of this
    limit_choices_to = {}

    def __init__(self, to):
        self._to = to

    @property
    def to(self):
        if not isinstance(self._to._meta, (DocumentMetaWrapper, LazyDocumentMetaWrapper)):
            self._to._meta = DocumentMetaWrapper(self._to)
        return self._to

    @to.setter
    def to(self, value):
        self._to = value

class PkWrapper(object):
    editable = False
    fake = False

    def __init__(self, wrapped):
        self.obj = wrapped

    def __getattr__(self, attr):
        if attr in dir(self.obj):
            return getattr(self.obj, attr)
        raise AttributeError

    def __setattr__(self, attr, value):
        if attr != 'obj' and hasattr(self.obj, attr):
            setattr(self.obj, attr, value)
        super(PkWrapper, self).__setattr__(attr, value)

class LazyDocumentMetaWrapper(LazyObject):
    _document = None
    _meta = None

    def __init__(self, document):
        self._document = document
        self._meta = document._meta
        super(LazyDocumentMetaWrapper, self).__init__()

    def _setup(self):
        self._wrapped = DocumentMetaWrapper(self._document, self._meta)

    def __setattr__(self, name, value):
        if name in ["_document", "_meta", ]:
            object.__setattr__(self, name, value)
        else:
            super(LazyDocumentMetaWrapper, self).__setattr__(name, value)

    __len__ = new_method_proxy(len)

    @new_method_proxy
    def __contains__(self, key):
        return key in self

class DocumentMetaWrapper(MutableMapping):
    """
    Used to store mongoengine's _meta dict to make the document admin
    as compatible as possible to django's meta class on models.
    """
    # attributes Django deprecated. Not really sure when to remove them
    _deprecated_attrs = {'module_name': 'model_name'}

    pk = None
    pk_name = None
    _app_label = None
    model_name = None
    _verbose_name = None
    has_auto_field = False
    object_name = None
    proxy = []
    proxied_children = []
    parents = {}
    many_to_many = []
    _field_cache = None
    document = None
    _meta = None
    concrete_model = None
    concrete_managers = []
    virtual_fields = []
    auto_created = False

    def __init__(self, document, meta=None):
        super(DocumentMetaWrapper, self).__init__()

        self.document = document
        # used by Django to distinguish between abstract and concrete models
        # here for now always the document
        self.concrete_model = document
        if meta is None:
            meta = getattr(document, '_meta', {})
            if not 'app_label' in meta:
                meta['app_label'] = document.__module__.split('.')[0]
            if not 'app_config' in meta:
                # TODO: this needs improvements
                meta['app_config'] = dict()
            if not 'verbose_name' in meta:
                meta['app_config']['verbose_name'] = meta['app_label'].replace('_', ' ').capitalize()
            if isinstance(meta, LazyDocumentMetaWrapper):
                meta = meta._meta
        self._meta = meta

        try:
            self.object_name = self.document.__name__
        except AttributeError:
            self.object_name = self.document.__class__.__name__

        self.model_name = self.object_name.lower()

        # add the gluey stuff to the document and it's fields to make
        # everything play nice with Django
        self._setup_document_fields()
        # Setup self.pk if the document has an id_field in it's meta
        # if it doesn't have one it's an embedded document
        # if 'id_field' in self._meta:
        #    self.pk_name = self._meta['id_field']
        self._init_pk()

    def _setup_document_fields(self):
        for f in self.document._fields.values():
            # Yay, more glue. Django expects fields to have a couple attributes
            # at least in the admin, probably in more places.
            if not hasattr(f, 'rel'):
                # need a bit more for actual reference fields here
                if isinstance(f, ReferenceField):
                    # FIXME: Probably broken in Django 1.7
                    f.rel = Relation(f.document_type)
                    f.is_relation = True
                elif isinstance(f, ListField) and isinstance(f.field, ReferenceField):
                    # FIXME: Probably broken in Django 1.7
                    f.field.rel = Relation(f.field.document_type)
                    f.field.is_relation = True
                else:
                    f.many_to_many = None
                    f.many_to_one = None
                    f.one_to_many = None
                    f.one_to_one = None
                    f.related_model = None

                    # FIXME: No longer used in Django 1.7?
                    f.rel = None
                    f.is_relation = False
            if not hasattr(f, 'verbose_name') or f.verbose_name is None:
                f.verbose_name = capfirst(create_verbose_name(f.name))
            if not hasattr(f, 'flatchoices'):
                flat = []
                if f.choices is not None:
                    for choice, value in f.choices:
                        if isinstance(value, (list, tuple)):
                            flat.extend(value)
                        else:
                            flat.append((choice, value))
                f.flatchoices = flat
            if isinstance(f, ReferenceField) and not \
                    isinstance(f.document_type._meta, (DocumentMetaWrapper, LazyDocumentMetaWrapper)) and \
                    self.document != f.document_type:
                f.document_type._meta = LazyDocumentMetaWrapper(f.document_type)
            if not hasattr(f, 'auto_created'):
                f.auto_created = False

    def _init_pk(self):
        """
        Adds a wrapper around the documents pk field. The wrapper object gets
        the attributes django expects on the pk field, like name and attname.

        The function also adds a _get_pk_val method to the document.
        """
        if 'id_field' in self._meta:
            self.pk_name = self._meta['id_field']
            pk_field = getattr(self.document, self.pk_name)
        else:
            pk_field = None
        self.pk = PkWrapper(pk_field)

        def _get_pk_val(obj):
            return obj.pk
        patch_document(_get_pk_val, self.document, False)  # document is a class...

        if pk_field is not None:
            self.pk.name = self.pk_name
            self.pk.attname = self.pk_name
        else:
            self.pk.fake = True
            # this is used in the admin and used to determine if the admin
            # needs to add a hidden pk field. It does not for embedded fields.
            # So we pretend to have an editable pk field and just ignore it otherwise
            self.pk.editable = True

    @property
    def app_label(self):
        if self._app_label is None:
            if self._meta.get('app_label'):
                self._app_label = self._meta["app_label"]
            else:
                model_module = sys.modules[self.document.__module__]
                self._app_label = model_module.__name__.split('.')[-2]
        return self._app_label

    @property
    def verbose_name(self):
        """
        Returns the verbose name of the document.

        Checks the original meta dict first. If it is not found
        then generates a verbose name from the object name.
        """
        if self._verbose_name is None:
            verbose_name = self._meta.get('verbose_name', self.object_name)
            self._verbose_name = capfirst(create_verbose_name(verbose_name))
        return self._verbose_name

    @property
    def verbose_name_raw(self):
        return self.verbose_name

    @property
    def verbose_name_plural(self):
        return "%ss" % self.verbose_name

    def get_add_permission(self):
        return 'add_%s' % self.object_name.lower()

    def get_change_permission(self):
        return 'change_%s' % self.object_name.lower()

    def get_delete_permission(self):
        return 'delete_%s' % self.object_name.lower()

    def get_ordered_objects(self):
        return []

    def get_field_by_name(self, name):
        """
        Returns the (field_object, model, direct, m2m), where field_object is
        the Field instance for the given name, model is the model containing
        this field (None for local fields), direct is True if the field exists
        on this model, and m2m is True for many-to-many relations. When
        'direct' is False, 'field_object' is the corresponding RelatedObject
        for this field (since the field doesn't have an instance associated
        with it).
        """
        if name in self.document._fields:
            field = self.document._fields[name]
            if isinstance(field, ReferenceField):
                return (field, field.document_type, False, False)
            else:
                return (field, None, True, False)
        else:
            raise FieldDoesNotExist('%s has no field named %r' %
                                    (self.object_name, name))

    def get_field(self, name, many_to_many=True):
        """
        Returns the requested field by name. Raises FieldDoesNotExist on error.
        """
        return self.get_field_by_name(name)[0]

    def get_fields(self, include_hidden=False):
        return self.document._fields.values()

    @property
    def swapped(self):
        """
        Has this model been swapped out for another? If so, return the model
        name of the replacement; otherwise, return None.

        For historical reasons, model name lookups using get_model() are
        case insensitive, so we make sure we are case insensitive here.

        NOTE: Not sure this is actually usefull for documents. So at the
        moment it's really only here because the admin wants it. It might
        prove usefull for someone though, so it's more then just a dummy.
        """
        if self._meta.get('swappable', False):
            model_label = '%s.%s' % (self.app_label, self.object_name.lower())
            swapped_for = getattr(settings, self.swappable, None)
            if swapped_for:
                try:
                    swapped_label, swapped_object = swapped_for.split('.')
                except ValueError:
                    # setting not in the format app_label.model_name
                    # raising ImproperlyConfigured here causes problems with
                    # test cleanup code - instead it is raised in
                    # get_user_model or as part of validation.
                    return swapped_for

                if '%s.%s' % (swapped_label, swapped_object.lower()) \
                        not in (None, model_label):
                    return swapped_for
        return None

    def __getattr__(self, name):
        if name in self._deprecated_attrs:
            return getattr(self, self._deprecated_attrs.get(name))

        try:
            return self._meta[name]
        except KeyError:
            raise AttributeError

    def __setattr__(self, name, value):
        if not hasattr(self, name):
            self._meta[name] = value
        else:
            super(DocumentMetaWrapper, self).__setattr__(name, value)

    def __contains__(self, key):
        return key in self._meta

    def __getitem__(self, key):
        return self._meta[key]

    def __setitem__(self, key, value):
        self._meta[key] = value

    def __delitem__(self, key):
        return self._meta.__delitem__(key)

    def __iter__(self):
        return self._meta.__iter__()

    def __len__(self):
        return self._meta.__len__()

    def __cmp__(self, other):
        return hash(self) == hash(other)

    def __hash__(self):
        return id(self)

    def get(self, key, default=None):
        try:
            return self.__getitem__(key)
        except KeyError:
            return default

    def get_parent_list(self):
        return []

    def get_all_related_objects(self, *args, **kwargs):
        return []

    def iteritems(self):
        return iter(self._meta.items())
<commit_msg>fix get model attribute for django 1.10<commit_after>import sys
from collections import MutableMapping
from types import MethodType

from django.db.models.fields import FieldDoesNotExist
from django.utils.text import capfirst
from django.utils.functional import LazyObject, new_method_proxy
from django.apps import apps
try:
    # New in Django 1.7+
    from django.utils.text import camel_case_to_spaces
except ImportError:
    # Backwards compatibility
    from django.db.models.options import get_verbose_name as camel_case_to_spaces
from django.conf import settings
from mongoengine.fields import ReferenceField, ListField

def patch_document(function, instance, bound=True):
    if bound:
        method = MethodType(function, instance)
    else:
        method = function
    setattr(instance, function.__name__, method)

def create_verbose_name(name):
    name = camel_case_to_spaces(name)
    name = name.replace('_', ' ')
    return name

class Relation(object):
    # just an empty dict to make it usable with Django
    # mongoengine has no notion of this
    limit_choices_to = {}

    def __init__(self, to):
        self._to = to

    @property
    def to(self):
        if not isinstance(self._to._meta, (DocumentMetaWrapper, LazyDocumentMetaWrapper)):
            self._to._meta = DocumentMetaWrapper(self._to)
        return self._to

    @to.setter
    def to(self, value):
        self._to = value

class PkWrapper(object):
    editable = False
    fake = False

    def __init__(self, wrapped):
        self.obj = wrapped

    def __getattr__(self, attr):
        if attr in dir(self.obj):
            return getattr(self.obj, attr)
        raise AttributeError

    def __setattr__(self, attr, value):
        if attr != 'obj' and hasattr(self.obj, attr):
            setattr(self.obj, attr, value)
        super(PkWrapper, self).__setattr__(attr, value)

class LazyDocumentMetaWrapper(LazyObject):
    _document = None
    _meta = None

    def __init__(self, document):
        self._document = document
        self._meta = document._meta
        super(LazyDocumentMetaWrapper, self).__init__()

    def _setup(self):
        self._wrapped = DocumentMetaWrapper(self._document, self._meta)

    def __setattr__(self, name, value):
        if name in ["_document", "_meta", ]:
            object.__setattr__(self, name, value)
        else:
            super(LazyDocumentMetaWrapper, self).__setattr__(name, value)

    __len__ = new_method_proxy(len)

    @new_method_proxy
    def __contains__(self, key):
        return key in self

class DocumentMetaWrapper(MutableMapping):
    """
    Used to store mongoengine's _meta dict to make the document admin
    as compatible as possible to django's meta class on models.
    """
    # attributes Django deprecated. Not really sure when to remove them
    _deprecated_attrs = {'module_name': 'model_name'}

    pk = None
    pk_name = None
    _app_label = None
    _app_config = None
    model_name = None
    _verbose_name = None
    has_auto_field = False
    object_name = None
    proxy = []
    proxied_children = []
    parents = {}
    many_to_many = []
    _field_cache = None
    document = None
    _meta = None
    concrete_model = None
    concrete_managers = []
    virtual_fields = []
    auto_created = False

    def __init__(self, document, meta=None):
        super(DocumentMetaWrapper, self).__init__()

        self.document = document
        # used by Django to distinguish between abstract and concrete models
        # here for now always the document
        self.concrete_model = document
        if meta is None:
            meta = getattr(document, '_meta', {})
            if 'app_label' not in meta:
                meta['app_label'] = document.__module__.split('.')[0]
            if 'app_config' not in meta:
                meta['app_config'] = apps.get_app_config(meta['app_label'])
            if 'model' not in meta:
                meta['model'] = self.document
            if isinstance(meta, LazyDocumentMetaWrapper):
                meta = meta._meta
        self._meta = meta

        try:
            self.object_name = self.document.__name__
        except AttributeError:
            self.object_name = self.document.__class__.__name__

        self.model_name = self.object_name.lower()

        # add the gluey stuff to the document and it's fields to make
        # everything play nice with Django
        self._setup_document_fields()
        # Setup self.pk if the document has an id_field in it's meta
        # if it doesn't have one it's an embedded document
        # if 'id_field' in self._meta:
        #    self.pk_name = self._meta['id_field']
        self._init_pk()

    def _setup_document_fields(self):
        for f in self.document._fields.values():
            # Yay, more glue. Django expects fields to have a couple attributes
            # at least in the admin, probably in more places.
            if not hasattr(f, 'rel'):
                # need a bit more for actual reference fields here
                if isinstance(f, ReferenceField):
                    # FIXME: Probably broken in Django 1.7
                    f.rel = Relation(f.document_type)
                    f.is_relation = True
                elif isinstance(f, ListField) and isinstance(f.field, ReferenceField):
                    # FIXME: Probably broken in Django 1.7
                    f.field.rel = Relation(f.field.document_type)
                    f.field.is_relation = True
                else:
                    f.many_to_many = None
                    f.many_to_one = None
                    f.one_to_many = None
                    f.one_to_one = None
                    f.related_model = None

                    # FIXME: No longer used in Django 1.7?
                    f.rel = None
                    f.is_relation = False
            if not hasattr(f, 'verbose_name') or f.verbose_name is None:
                f.verbose_name = capfirst(create_verbose_name(f.name))
            if not hasattr(f, 'flatchoices'):
                flat = []
                if f.choices is not None:
                    for choice, value in f.choices:
                        if isinstance(value, (list, tuple)):
                            flat.extend(value)
                        else:
                            flat.append((choice, value))
                f.flatchoices = flat
            if isinstance(f, ReferenceField) and not \
                    isinstance(f.document_type._meta, (DocumentMetaWrapper, LazyDocumentMetaWrapper)) and \
                    self.document != f.document_type:
                f.document_type._meta = LazyDocumentMetaWrapper(f.document_type)
            if not hasattr(f, 'auto_created'):
                f.auto_created = False

    def _init_pk(self):
        """
        Adds a wrapper around the documents pk field. The wrapper object gets
        the attributes django expects on the pk field, like name and attname.

        The function also adds a _get_pk_val method to the document.
        """
        if 'id_field' in self._meta:
            self.pk_name = self._meta['id_field']
            pk_field = getattr(self.document, self.pk_name)
        else:
            pk_field = None
        self.pk = PkWrapper(pk_field)

        def _get_pk_val(obj):
            return obj.pk
        patch_document(_get_pk_val, self.document, False)  # document is a class...

        if pk_field is not None:
            self.pk.name = self.pk_name
            self.pk.attname = self.pk_name
        else:
            self.pk.fake = True
            # this is used in the admin and used to determine if the admin
            # needs to add a hidden pk field. It does not for embedded fields.
            # So we pretend to have an editable pk field and just ignore it otherwise
            self.pk.editable = True

    @property
    def app_label(self):
        if self._app_label is None:
            if self._meta.get('app_label'):
                self._app_label = self._meta["app_label"]
            else:
                model_module = sys.modules[self.document.__module__]
                self._app_label = model_module.__name__.split('.')[-2]
        return self._app_label

    @property
    def app_config(self):
        if self._app_config is None:
            self._app_config = self._meta['app_config']
        return self._app_config

    @property
    def verbose_name(self):
        """
        Returns the verbose name of the document.

        Checks the original meta dict first. If it is not found
        then generates a verbose name from the object name.
        """
        if self._verbose_name is None:
            verbose_name = self._meta.get('verbose_name', self.object_name)
            self._verbose_name = capfirst(create_verbose_name(verbose_name))
        return self._verbose_name

    @property
    def verbose_name_raw(self):
        return self.verbose_name

    @property
    def verbose_name_plural(self):
        return "%ss" % self.verbose_name

    def get_add_permission(self):
        return 'add_%s' % self.object_name.lower()

    def get_change_permission(self):
        return 'change_%s' % self.object_name.lower()

    def get_delete_permission(self):
        return 'delete_%s' % self.object_name.lower()

    def get_ordered_objects(self):
        return []

    def get_field_by_name(self, name):
        """
        Returns the (field_object, model, direct, m2m), where field_object is
        the Field instance for the given name, model is the model containing
        this field (None for local fields), direct is True if the field exists
        on this model, and m2m is True for many-to-many relations. When
        'direct' is False, 'field_object' is the corresponding RelatedObject
        for this field (since the field doesn't have an instance associated
        with it).
        """
        if name in self.document._fields:
            field = self.document._fields[name]
            if isinstance(field, ReferenceField):
                return (field, field.document_type, False, False)
            else:
                return (field, None, True, False)
        else:
            raise FieldDoesNotExist('%s has no field named %r' %
                                    (self.object_name, name))

    def get_field(self, name, many_to_many=True):
        """
        Returns the requested field by name. Raises FieldDoesNotExist on error.
        """
        return self.get_field_by_name(name)[0]

    def get_fields(self, include_hidden=False):
        return self.document._fields.values()

    @property
    def swapped(self):
        """
        Has this model been swapped out for another? If so, return the model
        name of the replacement; otherwise, return None.

        For historical reasons, model name lookups using get_model() are
        case insensitive, so we make sure we are case insensitive here.

        NOTE: Not sure this is actually usefull for documents. So at the
        moment it's really only here because the admin wants it. It might
        prove usefull for someone though, so it's more then just a dummy.
        """
        if self._meta.get('swappable', False):
            model_label = '%s.%s' % (self.app_label, self.object_name.lower())
            swapped_for = getattr(settings, self.swappable, None)
            if swapped_for:
                try:
                    swapped_label, swapped_object = swapped_for.split('.')
                except ValueError:
                    # setting not in the format app_label.model_name
                    # raising ImproperlyConfigured here causes problems with
                    # test cleanup code - instead it is raised in
                    # get_user_model or as part of validation.
                    return swapped_for

                if '%s.%s' % (swapped_label, swapped_object.lower()) \
                        not in (None, model_label):
                    return swapped_for
        return None

    def __getattr__(self, name):
        if name in self._deprecated_attrs:
            return getattr(self, self._deprecated_attrs.get(name))
        try:
            return self._meta[name]
        except KeyError:
            raise AttributeError('Meta does not have attribute %s', name)

    def __setattr__(self, name, value):
        if not hasattr(self, name):
            self._meta[name] = value
        else:
            super(DocumentMetaWrapper, self).__setattr__(name, value)

    def __contains__(self, key):
        return key in self._meta

    def __getitem__(self, key):
        return self._meta[key]

    def __setitem__(self, key, value):
        self._meta[key] = value

    def __delitem__(self, key):
        return self._meta.__delitem__(key)

    def __iter__(self):
        return self._meta.__iter__()

    def __len__(self):
        return self._meta.__len__()

    def __cmp__(self, other):
        return hash(self) == hash(other)

    def __hash__(self):
        return id(self)

    def get(self, key, default=None):
        try:
            return self.__getitem__(key)
        except KeyError:
            return default

    def get_parent_list(self):
        return []

    def get_all_related_objects(self, *args, **kwargs):
        return []

    def iteritems(self):
        return iter(self._meta.items())
<|endoftext|>


<commit_before>from multiprocessing import Pool
from .base import Scorer

from django.conf import settings

from yelpapi import YelpAPI
from geopy.geocoders import GoogleV3
from geopy.distance import vincenty
import math

RADIUS = 1500
BORING_CHAINS = set(['Chipotle Mexican Grill'])
CHAIN_RADIUS = 3000

def sigmoid(x):
	return 1 / (1 + math.exp(-x))

class BusinessData(Scorer):
	"""
	Uses the Yelp api and geocoding to get and
	format business data the way we want.
	"""
	yelp = YelpAPI(settings.CONSUMER_KEY, settings.CONSUMER_SECRET, settings.TOKEN, settings.TOKEN_SECRET)
	geolocator = GoogleV3(api_key=settings.GOOGLE_MAPS_API_KEY)

	def get_chains_near(self, latitude, longitude, radius, term='burrito', category_filter='mexican'):
		"""
		Gets formatted businesses near the given location within the radius.
		"""
		# Search by lat/lng
		search_results = self.yelp.search_query(
			ll='%s,%s' % (latitude, longitude),
			term='Chipotle',
			category_filter=category_filter,
			radius_filter=radius,
		)

		businesses = search_results['businesses']
		businesses = self._format_data((latitude, longitude), businesses)

		return businesses

	def replacement_burrito(self, latitude, longitude):
		businesses = self.get_chains_near(latitude, longitude, CHAIN_RADIUS)
		scores = [business['rating'] for business in businesses.values() if business['name'] in BORING_CHAINS]
		if len(scores) > 0:
			return sum(scores) / len(scores)
		else:
			return 3.5

	def score(self, location):
		# Geolocate the given location
		address, (latitude, longitude) = self.geolocator.geocode(location)
		replacement_burrito = self.replacement_burrito(latitude, longitude)
		print "replacement burrito = ", replacement_burrito

		businesses = self.get_all_near(latitude, longitude, RADIUS)
		for business in businesses.values():
			business['vorb'] = business['rating'] - replacement_burrito
		score = 0.0
		for (multiplier, vorb_min, vorb_max) in (
			(1.0, 0, 0.24),
			(10.0, 0.25, 0.74),
			(15.0, 0.75, 1.24),
			(20.0, 1.25, 5),
		):
			den = self.density(vorb_min, vorb_max, businesses.values())
			print "density of ", vorb_min, "to", vorb_max, "=", den
			score += multiplier * den

		return (round(score, 0), RADIUS, businesses)

	def density(self, vorb_min, vorb_max, businesses):
		distances = [business['distance'] for business in businesses if business['vorb'] <= vorb_max and business['vorb'] >= vorb_min]
		if len(distances) == 0:
			return 0
		else:
			unsquashed = (len(distances) / sum(distances)) * 1000
			squashed = sigmoid(unsquashed)
			return squashed

	def get_all_near(self, latitude, longitude, radius, term='burrito', category_filter='mexican'):
		"""
		Gets formatted businesses near the given location within the radius.
		"""

		# Search by lat/lng
		search_results = self.yelp.search_query(
			ll='%s,%s' % (latitude, longitude),
			term=term,
			category_filter=category_filter,
			radius_filter=radius,
		)

		businesses = search_results['businesses']
		businesses = self._format_data((latitude, longitude), businesses)

		return businesses

	def _format_data(self, location, businesses):
		"""
		Things we care about: name, location, distance, rating, review_count
		"""
		# Get all the missing data we need
		pool = Pool()
		geolocated_businesses = pool.map(geolocate_business, businesses)
		backfilled_businesses = self._backfill_distances(geolocated_businesses)

		# Only keep the stuff we care about
		formatted_businesses = {}
		for business in backfilled_businesses:
			formatted_businesses[business['id']] = {
				'name': business['name'],
				'address': ' '.join(business['location']['address']),
				'lat': business['location']['coordinate']['latitude'],
				'lon': business['location']['coordinate']['longitude'],
				'distance': business['distance'],
				'rating': business['rating'],
				'review_count': business['review_count'],
			}

		return formatted_businesses

	def _backfill_distances(location, businesses):
		for business in businesses:
			if 'distance' not in business:
				business['distance'] = vincenty(location, (latitude, longitude)).meters

		return businesses

def geolocate_business(business):
	"""
	Take Yelp businesses and backfill with geodata for those missing it.
	"""
	# backfill coordinate data if necessary
	if 'coordinate' not in business['location']:
		geolocator = GoogleV3(api_key=settings.GOOGLE_MAPS_API_KEY)
		business_address = ' '.join(business['location']['display_address'])
		address, (latitude, longitude) = geolocator.geocode(business_address)
		business['location']['coordinate'] = {'latitude': latitude, 'longitude': longitude}

	return business

<commit_msg>Make 0-100 scale<commit_after>from multiprocessing import Pool
from .base import Scorer

from django.conf import settings

from yelpapi import YelpAPI
from geopy.geocoders import GoogleV3
from geopy.distance import vincenty
import math

RADIUS = 1500
BORING_CHAINS = set(['Chipotle Mexican Grill'])
CHAIN_RADIUS = 3000

def sigmoid(x):
	return 1 / (1 + math.exp(-x))

class BusinessData(Scorer):
	"""
	Uses the Yelp api and geocoding to get and
	format business data the way we want.
	"""
	yelp = YelpAPI(settings.CONSUMER_KEY, settings.CONSUMER_SECRET, settings.TOKEN, settings.TOKEN_SECRET)
	geolocator = GoogleV3(api_key=settings.GOOGLE_MAPS_API_KEY)

	def get_chains_near(self, latitude, longitude, radius, term='burrito', category_filter='mexican'):
		"""
		Gets formatted businesses near the given location within the radius.
		"""
		# Search by lat/lng
		search_results = self.yelp.search_query(
			ll='%s,%s' % (latitude, longitude),
			term='Chipotle',
			category_filter=category_filter,
			radius_filter=radius,
		)

		businesses = search_results['businesses']
		businesses = self._format_data((latitude, longitude), businesses)

		return businesses

	def replacement_burrito(self, latitude, longitude):
		businesses = self.get_chains_near(latitude, longitude, CHAIN_RADIUS)
		scores = [business['rating'] for business in businesses.values() if business['name'] in BORING_CHAINS]
		if len(scores) > 0:
			return sum(scores) / len(scores)
		else:
			return 3.5

	def score(self, location):
		# Geolocate the given location
		address, (latitude, longitude) = self.geolocator.geocode(location)
		replacement_burrito = self.replacement_burrito(latitude, longitude)
		print "replacement burrito = ", replacement_burrito

		businesses = self.get_all_near(latitude, longitude, RADIUS)
		for business in businesses.values():
			business['vorb'] = business['rating'] - replacement_burrito
		score = 0.0
		for (multiplier, vorb_min, vorb_max) in (
			(1.0, 0, 0.24),
            (14.0, 0.25, 0.49),
			(20.0, 0.50, 0.74),
			(25.0, 0.75, 1.24),
			(40.0, 1.25, 5),
		):
			den = self.density(vorb_min, vorb_max, businesses.values())
			print "density of ", vorb_min, "to", vorb_max, "=", den
			score += multiplier * den

		return (round(score, 0), RADIUS, businesses)

	def density(self, vorb_min, vorb_max, businesses):
		distances = [business['distance'] for business in businesses if business['vorb'] <= vorb_max and business['vorb'] >= vorb_min]
		if len(distances) == 0:
			return 0
		else:
			unsquashed = (len(distances) / sum(distances)) * 1000
			squashed = sigmoid(unsquashed)
			return squashed

	def get_all_near(self, latitude, longitude, radius, term='burrito', category_filter='mexican'):
		"""
		Gets formatted businesses near the given location within the radius.
		"""

		# Search by lat/lng
		search_results = self.yelp.search_query(
			ll='%s,%s' % (latitude, longitude),
			term=term,
			category_filter=category_filter,
			radius_filter=radius,
		)

		businesses = search_results['businesses']
		businesses = self._format_data((latitude, longitude), businesses)

		return businesses

	def _format_data(self, location, businesses):
		"""
		Things we care about: name, location, distance, rating, review_count
		"""
		# Get all the missing data we need
		pool = Pool()
		geolocated_businesses = pool.map(geolocate_business, businesses)
		backfilled_businesses = self._backfill_distances(geolocated_businesses)

		# Only keep the stuff we care about
		formatted_businesses = {}
		for business in backfilled_businesses:
			formatted_businesses[business['id']] = {
				'name': business['name'],
				'address': ' '.join(business['location']['address']),
				'lat': business['location']['coordinate']['latitude'],
				'lon': business['location']['coordinate']['longitude'],
				'distance': business['distance'],
				'rating': business['rating'],
				'review_count': business['review_count'],
			}

		return formatted_businesses

	def _backfill_distances(location, businesses):
		for business in businesses:
			if 'distance' not in business:
				business['distance'] = vincenty(location, (latitude, longitude)).meters

		return businesses

def geolocate_business(business):
	"""
	Take Yelp businesses and backfill with geodata for those missing it.
	"""
	# backfill coordinate data if necessary
	if 'coordinate' not in business['location']:
		geolocator = GoogleV3(api_key=settings.GOOGLE_MAPS_API_KEY)
		business_address = ' '.join(business['location']['display_address'])
		address, (latitude, longitude) = geolocator.geocode(business_address)
		business['location']['coordinate'] = {'latitude': latitude, 'longitude': longitude}

	return business

<|endoftext|>