<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# import numpy as np
import csv
import logging
import pandas as pd
from sklearn.linear_model.logistic import LogisticRegression
from sklearn.grid_search import GridSearchCV
# from sklearn.pipeline import Pipeline
import sys
reload(sys)
sys.setdefaultencoding('utf8')

logger = logging.getLogger('aucLog')
logger.setLevel(logging.INFO)

logfile = logging.FileHandler('auc.log')
logfile.setLevel(logging.INFO)

formatter = logging.Formatter('等价于sklearn.metrics.roc_auc_score')
logfile.setFormatter(formatter)

logger.addHandler(logfile)

def get_field(raw_data):
    # 提取所需字段
    field = pd.DataFrame({
        '年龄段': raw_data['年龄段'],
        '大致消费水平': raw_data['大致消费水平'],
        '每月的大致刷卡消费次数': raw_data['每月的大致刷卡消费次数'],
        '爱奇艺': raw_data['爱奇艺'],
        '腾讯视频': raw_data['腾讯视频'],
        '唱吧': raw_data['唱吧'],
        '女孩相机': raw_data['女孩相机'],
        '什么值得买': raw_data['什么值得买'],
        '星巴克中国': raw_data['星巴克中国'],
        '亚马逊': raw_data['亚马逊'],
        '蘑菇街': raw_data['蘑菇街'],
        '天猫': raw_data['天猫'],
        '手机天猫': raw_data['手机天猫'],
        '苹果地图': raw_data['苹果地图'],
        '嘀嘀打车': raw_data['嘀嘀打车'],
        '高铁管家': raw_data['高铁管家'],
        '航旅纵横': raw_data['航旅纵横'],
        '虎扑看球': raw_data['虎扑看球'],
        '新浪微博': raw_data['新浪微博'],
        '雪球': raw_data['雪球'],
        '百度魔图': raw_data['百度魔图'],
        '天天P图': raw_data['天天P图'],
        'Safari': raw_data['Safari'],
        'HUAWEI Browser': raw_data['HUAWEI Browser'],
        'XiaoMi Browser': raw_data['XiaoMi Browser'],
        'OPPO Browser': raw_data['OPPO Browser'],
        'SAMSUNG Browser': raw_data['SAMSUNG Browser'],
        'VIVO Browser': raw_data['VIVO Browser'],
        '华为应用市场': raw_data['华为应用市场'],
        '豆瓣': raw_data['豆瓣'],
        'VIVAME': raw_data['VIVAME'],
        'zaker': raw_data['zaker'],
        '苹果iphone股票': raw_data['苹果iphone股票'],
        'QQ音乐': raw_data['QQ音乐'],
        '百度音乐': raw_data['百度音乐'],
        '酷狗音乐': raw_data['酷狗音乐'],
        '酷我音乐': raw_data['酷我音乐'],
        '网易云音乐': raw_data['网易云音乐'],
        '虾米音乐': raw_data['虾米音乐'],
        '开心消消乐': raw_data['开心消消乐'],
        '天天爱消除': raw_data['天天爱消除'],
        '炉石传说': raw_data['炉石传说'],
        'FT中文网': raw_data['FT中文网'],
        'iTunes Store': raw_data['iTunes Store'],
        '百度外卖': raw_data['百度外卖'],
        '嘀嗒拼车': raw_data['嘀嗒拼车'],
        '广发证券易淘金': raw_data['广发证券易淘金'],
        '美柚': raw_data['美柚'],
        '探探': raw_data['探探'],
        '携程旅行': raw_data['携程旅行'],
        '一点资讯': raw_data['一点资讯'],
        '易游人': raw_data['易游人'],
        '支付宝': raw_data['支付宝'],
        '中国工商银行': raw_data['中国工商银行'],
        '墨迹天气': raw_data['墨迹天气'],
        '美图秀秀': raw_data['美图秀秀'],
        '快的打车': raw_data['快的打车'],
        'appstore': raw_data['appstore'],
        '大麦': raw_data['大麦'],
        '京东': raw_data['京东'],
        '京东到家': raw_data['京东到家'],
        '手机淘宝': raw_data['手机淘宝'],
        '唯品会': raw_data['唯品会'],
        '百度糯米': raw_data['百度糯米'],
        '大众点评': raw_data['大众点评'],
        '饿了么': raw_data['饿了么'],
        '肯德基': raw_data['肯德基'],
        '百度地图': raw_data['百度地图'],
        '高德地图': raw_data['高德地图'],
        '阿里旅行': raw_data['阿里旅行'],
        '去哪儿旅行': raw_data['去哪儿旅行'],
        '同程旅游': raw_data['同程旅游'],
        '易到用车': raw_data['易到用车'],
        'QQ': raw_data['QQ'],
        '微信': raw_data['微信'],
        '陌陌': raw_data['陌陌'],
        '搜狗输入法': raw_data['搜狗输入法'],
        '讯飞输入法': raw_data['讯飞输入法'],
        '百度云': raw_data['百度云'],
        '今日头条': raw_data['今日头条'],
        '知乎': raw_data['知乎'],
        '有道词典': raw_data['有道词典'],
        '喜马拉雅': raw_data['喜马拉雅'],
        '大智慧免费炒股软件': raw_data['大智慧免费炒股软件'],
        '工行手机银行': raw_data['工行手机银行'],
        '农行掌上银行': raw_data['农行掌上银行'],
        '中国建设银行': raw_data['中国建设银行'],
        '固定联络圈规模': raw_data['固定联络圈规模'],
        '访问视频网站的次数': raw_data['访问视频网站的次数'],
        '访问音乐网站的次数': raw_data['访问音乐网站的次数'],
        '访问动漫网站的次数': raw_data['访问动漫网站的次数'],
        '访问搜索网站的次数': raw_data['访问搜索网站的次数'],
        '访问生活网站的次数': raw_data['访问生活网站的次数'],
        '访问购物网站的次数': raw_data['访问购物网站的次数'],
        '访问房产网站的次数': raw_data['访问房产网站的次数'],
        '访问地图网站的次数': raw_data['访问地图网站的次数'],
        '访问餐饮网站的次数': raw_data['访问餐饮网站的次数'],
        '访问旅游网站的次数': raw_data['访问旅游网站的次数'],
        '访问聊天网站的次数': raw_data['访问聊天网站的次数'],
        '访问交友网站的次数': raw_data['访问交友网站的次数'],
        '访问社交网站的次数': raw_data['访问社交网站的次数'],
        '访问问答网站的次数': raw_data['访问问答网站的次数'],
        '访问新闻网站的次数': raw_data['访问新闻网站的次数'],
        '访问金融网站的次数': raw_data['访问金融网站的次数'],
        '访问股票网站的次数': raw_data['访问股票网站的次数']})
    return field

def get_result(predict_result):
    '''得到预测结果概率
    Params:
        predict_results: 原始预测结果
    Returns: 精确到小数点后四位的结果
    '''
    return round(predict_result[1], 6)

def compute_auc(predict_file, real_results):
    '''计算AUC评分, 等价于 sklearn.metrics.roc_auc_score
    Params:
        predict_file: 预测结果文件
        real_results: 真实结果
    Returns:
        auc: AUC评分
    '''
    # N: number of negative; M: number of positive
    N, M = real_results['是否去过迪士尼'].value_counts()
    yes = real_results[real_results['是否去过迪士尼'] == 1]
    no = real_results[real_results['是否去过迪士尼'] == 0]
    score = []
    pf = pd.read_csv(predict_file)
    for i in yes['用户标识']:
        p = pf[pf['IMEI'] == i]['SCORE']
        for j in no['用户标识']:
            n = pf[pf['IMEI'] == j]['SCORE']
            if p.values[0] > n.values[0]:
                score.append(1)
            elif p.values[0] == n.values[0]:
                score.append(0.5)
            else:
                score.append(0)
    auc = sum(score) / (M * N)
    return auc

if __name__ == '__main__':
    data = pd.read_csv('/data/第1题：算法题数据/数据集1_用户标签_本地_训练集.csv')
    y_train = pd.read_csv('/data/第1题：算法题数据/用户是否去过迪士尼_训练集.csv')
    X_train = get_field(data)
    y_test = y_train[840000:]

    # 未使用手机品牌字段 counts = data['手机品牌'].value_counts()
    lr = ('clf', LogisticRegression())
    params = {
        'clf_penalty': ('l1', 'l2'),
        'clf_c': (0.01, 0.1, 1, 10)
    }
    grid_search = GridSearchCV(lr, params, verbose=1, scoring='roc_auc', cv=3)
    grid_search.fit(X_train[0:840000], y_train['是否去过迪士尼'][0:840000])
    logger.info('最佳效果' + str(grid_search.best_score_))

    results = grid_search.predict_proba(X_train[840000:])
    predict_results = map(get_result, results)

    # 预测结果写入文件
    with open('result.csv', 'wb') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['IMEI', 'SCORE'])
        for idx in range(len(X_train[840000:])):
            writer.writerow(
                [(data['用户标识'][840000:])[840000 + idx],
                 '%.6f' % predict_results[idx]])
    # 计算AUC评分
    logger.info(str(compute_auc('result.csv', y_test)))
<commit_msg>fix: 修改了error，可运行<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# import numpy as np
import csv
import logging
import pandas as pd
from sklearn.linear_model.logistic import LogisticRegression
from sklearn.grid_search import GridSearchCV
from sklearn.pipeline import Pipeline
import sys
reload(sys)
sys.setdefaultencoding('utf8')

logger = logging.getLogger('aucLog')
logger.setLevel(logging.INFO)

logfile = logging.FileHandler('auc.log')
logfile.setLevel(logging.INFO)

formatter = logging.Formatter('等价于sklearn.metrics.roc_auc_score')
logfile.setFormatter(formatter)

logger.addHandler(logfile)

def get_field(raw_data):
    # 提取所需字段
    field = pd.DataFrame({
        '年龄段': raw_data['年龄段'],
        '大致消费水平': raw_data['大致消费水平'],
        '每月的大致刷卡消费次数': raw_data['每月的大致刷卡消费次数'],
        '爱奇艺': raw_data['爱奇艺'],
        '腾讯视频': raw_data['腾讯视频'],
        '唱吧': raw_data['唱吧'],
        '女孩相机': raw_data['女孩相机'],
        '什么值得买': raw_data['什么值得买'],
        '星巴克中国': raw_data['星巴克中国'],
        '亚马逊': raw_data['亚马逊'],
        '蘑菇街': raw_data['蘑菇街'],
        '天猫': raw_data['天猫'],
        '手机天猫': raw_data['手机天猫'],
        '苹果地图': raw_data['苹果地图'],
        '嘀嘀打车': raw_data['嘀嘀打车'],
        '高铁管家': raw_data['高铁管家'],
        '航旅纵横': raw_data['航旅纵横'],
        '虎扑看球': raw_data['虎扑看球'],
        '新浪微博': raw_data['新浪微博'],
        '雪球': raw_data['雪球'],
        '百度魔图': raw_data['百度魔图'],
        '天天P图': raw_data['天天P图'],
        'Safari': raw_data['Safari'],
        'HUAWEI Browser': raw_data['HUAWEI Browser'],
        'XiaoMi Browser': raw_data['XiaoMi Browser'],
        'OPPO Browser': raw_data['OPPO Browser'],
        'SAMSUNG Browser': raw_data['SAMSUNG Browser'],
        'VIVO Browser': raw_data['VIVO Browser'],
        '华为应用市场': raw_data['华为应用市场'],
        '豆瓣': raw_data['豆瓣'],
        'VIVAME': raw_data['VIVAME'],
        'zaker': raw_data['zaker'],
        '苹果iphone股票': raw_data['苹果iphone股票'],
        'QQ音乐': raw_data['QQ音乐'],
        '百度音乐': raw_data['百度音乐'],
        '酷狗音乐': raw_data['酷狗音乐'],
        '酷我音乐': raw_data['酷我音乐'],
        '网易云音乐': raw_data['网易云音乐'],
        '虾米音乐': raw_data['虾米音乐'],
        '开心消消乐': raw_data['开心消消乐'],
        '天天爱消除': raw_data['天天爱消除'],
        '炉石传说': raw_data['炉石传说'],
        'FT中文网': raw_data['FT中文网'],
        'iTunes Store': raw_data['iTunes Store'],
        '百度外卖': raw_data['百度外卖'],
        '嘀嗒拼车': raw_data['嘀嗒拼车'],
        '广发证券易淘金': raw_data['广发证券易淘金'],
        '美柚': raw_data['美柚'],
        '探探': raw_data['探探'],
        '携程旅行': raw_data['携程旅行'],
        '一点资讯': raw_data['一点资讯'],
        '易游人': raw_data['易游人'],
        '支付宝': raw_data['支付宝'],
        '中国工商银行': raw_data['中国工商银行'],
        '墨迹天气': raw_data['墨迹天气'],
        '美图秀秀': raw_data['美图秀秀'],
        '快的打车': raw_data['快的打车'],
        'appstore': raw_data['appstore'],
        '大麦': raw_data['大麦'],
        '京东': raw_data['京东'],
        '京东到家': raw_data['京东到家'],
        '手机淘宝': raw_data['手机淘宝'],
        '唯品会': raw_data['唯品会'],
        '百度糯米': raw_data['百度糯米'],
        '大众点评': raw_data['大众点评'],
        '饿了么': raw_data['饿了么'],
        '肯德基': raw_data['肯德基'],
        '百度地图': raw_data['百度地图'],
        '高德地图': raw_data['高德地图'],
        '阿里旅行': raw_data['阿里旅行'],
        '去哪儿旅行': raw_data['去哪儿旅行'],
        '同程旅游': raw_data['同程旅游'],
        '易到用车': raw_data['易到用车'],
        'QQ': raw_data['QQ'],
        '微信': raw_data['微信'],
        '陌陌': raw_data['陌陌'],
        '搜狗输入法': raw_data['搜狗输入法'],
        '讯飞输入法': raw_data['讯飞输入法'],
        '百度云': raw_data['百度云'],
        '今日头条': raw_data['今日头条'],
        '知乎': raw_data['知乎'],
        '有道词典': raw_data['有道词典'],
        '喜马拉雅': raw_data['喜马拉雅'],
        '大智慧免费炒股软件': raw_data['大智慧免费炒股软件'],
        '工行手机银行': raw_data['工行手机银行'],
        '农行掌上银行': raw_data['农行掌上银行'],
        '中国建设银行': raw_data['中国建设银行'],
        '固定联络圈规模': raw_data['固定联络圈规模'],
        '访问视频网站的次数': raw_data['访问视频网站的次数'],
        '访问音乐网站的次数': raw_data['访问音乐网站的次数'],
        '访问动漫网站的次数': raw_data['访问动漫网站的次数'],
        '访问搜索网站的次数': raw_data['访问搜索网站的次数'],
        '访问生活网站的次数': raw_data['访问生活网站的次数'],
        '访问购物网站的次数': raw_data['访问购物网站的次数'],
        '访问房产网站的次数': raw_data['访问房产网站的次数'],
        '访问地图网站的次数': raw_data['访问地图网站的次数'],
        '访问餐饮网站的次数': raw_data['访问餐饮网站的次数'],
        '访问旅游网站的次数': raw_data['访问旅游网站的次数'],
        '访问聊天网站的次数': raw_data['访问聊天网站的次数'],
        '访问交友网站的次数': raw_data['访问交友网站的次数'],
        '访问社交网站的次数': raw_data['访问社交网站的次数'],
        '访问问答网站的次数': raw_data['访问问答网站的次数'],
        '访问新闻网站的次数': raw_data['访问新闻网站的次数'],
        '访问金融网站的次数': raw_data['访问金融网站的次数'],
        '访问股票网站的次数': raw_data['访问股票网站的次数']})
    return field

def get_result(predict_result):
    '''得到预测结果概率
    Params:
        predict_results: 原始预测结果
    Returns: 精确到小数点后四位的结果
    '''
    return round(predict_result[1], 6)

def compute_auc(predict_file, real_results):
    '''计算AUC评分, 等价于 sklearn.metrics.roc_auc_score
    Params:
        predict_file: 预测结果文件
        real_results: 真实结果
    Returns:
        auc: AUC评分
    '''
    # N: number of negative; M: number of positive
    N, M = real_results['是否去过迪士尼'].value_counts()
    yes = real_results[real_results['是否去过迪士尼'] == 1]
    no = real_results[real_results['是否去过迪士尼'] == 0]
    score = []
    pf = pd.read_csv(predict_file)
    for i in yes['用户标识']:
        p = pf[pf['IMEI'] == i]['SCORE']
        for j in no['用户标识']:
            n = pf[pf['IMEI'] == j]['SCORE']
            if p.values[0] > n.values[0]:
                score.append(1)
            elif p.values[0] == n.values[0]:
                score.append(0.5)
            else:
                score.append(0)
    auc = sum(score) / (M * N)
    return auc

if __name__ == '__main__':
    data = pd.read_csv('/data/第1题：算法题数据/数据集1_用户标签_本地_训练集.csv')
    y_train = pd.read_csv('/data/第1题：算法题数据/用户是否去过迪士尼_训练集.csv')
    X_train = get_field(data)
    y_test = y_train[840000:]

    # 未使用手机品牌字段 counts = data['手机品牌'].value_counts()
    pipeline = Pipeline([('clf', LogisticRegression())])
    params = {
        'clf__penalty': ('l1', 'l2'),
        'clf__C': (0.01, 0.1, 1, 10)
    }
    grid_search = GridSearchCV(pipeline,
                               params,
                               verbose=1,
                               scoring='roc_auc',
                               cv=3)
    grid_search.fit(X_train[0:840000], y_train['是否去过迪士尼'][0:840000])
    logger.info('最佳效果' + str(grid_search.best_score_))

    results = grid_search.predict_proba(X_train[840000:])
    predict_results = map(get_result, results)

    # 预测结果写入文件
    with open('result.csv', 'wb') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['IMEI', 'SCORE'])
        for idx in range(len(X_train[840000:])):
            writer.writerow(
                [(data['用户标识'][840000:])[840000 + idx],
                 '%.6f' % predict_results[idx]])
    # 计算AUC评分
    logger.info(str(compute_auc('result.csv', y_test)))
<|endoftext|>


<commit_before>import json
import logging
from .forms import StaffingForm
from .models import FireStation, Staffing, FireDepartment
from firecares.weather.models import DepartmentWarnings
from django.core.serializers.json import DjangoJSONEncoder
from django.contrib.gis import geos
from tastypie import fields
from tastypie.authentication import SessionAuthentication, ApiKeyAuthentication, MultiAuthentication, Authentication
from tastypie.authorization import DjangoAuthorization
from tastypie.cache import SimpleCache
from tastypie.constants import ALL
from tastypie.contrib.gis.resources import ModelResource
from tastypie.exceptions import Unauthorized, TastypieError
from tastypie.serializers import Serializer
from tastypie.validation import FormValidation
from guardian.core import ObjectPermissionChecker
from django.utils import timezone

logger = logging.getLogger(__name__)

# Note this is from one of my other projects, not sure it is actually needed or not.
class SessionAuth(SessionAuthentication):
    """
    This is a hack to fix a bug which returns occasional TypeErrors returned from SessionAuthentication.

    About:
    Every now and then the super class' get_identifier returns a TypeError (getattr(): attribute name must be string).
    It seems that the logic that returns the string used for the username sometimes returns None.

    """
    def get_identifier(self, request):
        """
        Provides a unique string identifier for the requestor.

        This implementation returns the user's username.
        """
        try:
            return super(SessionAuth, self).get_identifier(request)
        except TypeError:
            return getattr(request.user, 'username')

class PrettyJSONSerializer(Serializer):
    json_indent = 2

    def to_json(self, data, options=None):
        options = options or {}
        data = self.to_simple(data, options)
        return json.dumps(data, cls=DjangoJSONEncoder,
                          sort_keys=True, ensure_ascii=False, indent=self.json_indent)

class JSONDefaultModelResourceMixin(object):
    def determine_format(self, request):
        return 'application/json' if not request.GET.get('format') else super(JSONDefaultModelResourceMixin, self).determine_format(request)

class PropertyResolutionFailedException(TastypieError):
    def __init__(self, property_path, class_name, *args, **kwargs):
        msg = '"{}" does not resolve for class type {}'.format(property_path, class_name)
        super(PropertyResolutionFailedException, self).__init__(msg, *args, **kwargs)

# Pulled/modified from https://gist.github.com/7wonders/6557760
class GuardianAuthorization(DjangoAuthorization):
    """
    :create_permission_code:
        the permission code that signifies the user can create one of these objects
    :view_permission_code:
        the permission code that signifies the user can view the detail
    :update_permission_code:
        the permission code that signifies the user can update one of these objects
    :remove_permission_code:
        the permission code that signifies the user can remove one of these objects
    :kwargs:
        other permission codes
        class Something(models.Model):
            name = models.CharField()
        class SomethingResource(ModelResource):
            class Meta:
                queryset = Something.objects.all()
                authorization = GuardianAuthorization(
                    delegate_to_property = None,  # property path (dot notation) to delegate authorization to
                    view_permission_code = 'can_view',  # empty view_permission_code allows anonymous users to view
                    create_permission_code = 'can_create',
                    update_permission_code = 'can_update',
                    delete_permission_code = 'can_delete'
                    )
    """

    def __init__(self, *args, **kwargs):
        # Allow for authorization to be delegated through a property on the object
        self.parent_property = kwargs.pop('delegate_to_property', None)
        self.view_permission_code = kwargs.pop("view_permission_code", 'can_view')
        self.create_permission_code = kwargs.pop("create_permission_code", 'can_create')
        self.update_permission_code = kwargs.pop("update_permission_code", 'can_update')
        self.delete_permission_code = kwargs.pop("delete_permission_code", 'can_delete')
        super(GuardianAuthorization, self).__init__(**kwargs)

    def resolve_property(self, obj, property_path):
        path = property_path.split('.')
        ret = obj
        for p in path:
            if hasattr(ret, p):
                ret = getattr(ret, p)
            else:
                raise PropertyResolutionFailedException(property_path, obj.__class__)
        return ret

    def generic_base_check(self, object_list, bundle):
        """
            Returns False if either:
                a) if the `object_list.model` doesn't have a `_meta` attribute
                b) the `bundle.request` object doesn have a `user` attribute
        """
        klass = self.base_checks(bundle.request, object_list.model)
        if klass is False:
            raise Unauthorized("You are not allowed to access that resource.")
        return True

    def generic_item_check(self, object_list, bundle, permission):
        if bundle.request.user.is_superuser:
            return True

        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")

        checker = ObjectPermissionChecker(bundle.request.user)
        if self.parent_property:
            obj = self.resolve_property(bundle.obj, self.parent_property)
        else:
            obj = bundle.obj

        # When the property resolves to None, prevent access by default
        if obj is None or not checker.has_perm(permission, obj):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    def generic_list_check(self, object_list, bundle, permission):
        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")
        user = bundle.request.user
        return [i for i in object_list if permission is None or user.has_perm(permission, i) or user.is_superuser]

    def generic_post_check(self, object_list, bundle, permission):
        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")
        user = bundle.request.user
        if not user.has_perm(permission):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    # List Checks
    def create_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.create_permission_code)

    def read_list(self, object_list, bundle):
        if not self.view_permission_code:
            return object_list

        return self.generic_list_check(object_list, bundle,
                                       self.view_permission_code)

    def update_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.update_permission_code)

    def delete_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.delete_permission_code)

    # Item Checks
    def create_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.create_permission_code)

    def read_detail(self, object_list, bundle):
        if not self.view_permission_code:
            return True
        return self.generic_item_check(object_list, bundle,
                                       self.view_permission_code)

    def update_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.update_permission_code)

    def delete_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.delete_permission_code)

class FireDepartmentResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Fire Department API.
    """

    class Meta:
        readonly_fields = ['owned_tracts_geom']
        resource_name = 'fire-departments'
        queryset = FireDepartment.objects.defer('owned_tracts_geom').filter(archived=False)
        authorization = GuardianAuthorization(view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='admin_firedepartment')
        authentication = MultiAuthentication(SessionAuthentication(), ApiKeyAuthentication())
        cache = SimpleCache()
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get', 'put']
        filtering = {'state': ALL, 'featured': ALL}
        serializer = PrettyJSONSerializer()
        limit = 120
        max_limit = 2000

    def __init__(self, **kwargs):
        super(FireDepartmentResource, self).__init__(**kwargs)
        for f in getattr(self.Meta, 'readonly_fields', []):
            self.fields[f].readonly = True

    def hydrate_geom(self, bundle):
        geom = bundle.data.get('geom')
        boundary = geos.GEOSGeometry(json.dumps(geom))
        if type(boundary) is geos.Polygon:
            bundle.data['geom'] = json.loads(geos.MultiPolygon(boundary).json)
        return bundle

    def dehydrate(self, bundle):
        """
        remove fields not requested

        note: this is called for every object, so works for both detail and
              list requests
        """
        only_fields = bundle.request.GET.get('fields')
        debug_fields = bundle.request.GET.get('fields_debug', False)
        if only_fields:
            only_fields = only_fields.split(',')
            fields_to_remove = [field for field in bundle.data.keys()
                                if field not in only_fields]
            for field in fields_to_remove:
                del bundle.data[field]
            if debug_fields:
                bundle.data['_fields_selected'] = [field for field in only_fields
                                                   if field in bundle.data.keys()]
                bundle.data['_fields_removed'] = fields_to_remove
        return bundle

class FireStationResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Fire Station API.
    """

    department = fields.ForeignKey(FireDepartmentResource, 'department', null=True)

    class Meta:
        resource_name = 'firestations'

        queryset = FireStation.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='admin_firedepartment')
        authentication = MultiAuthentication(Authentication(), SessionAuthentication(), ApiKeyAuthentication())
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get', 'put']
        filtering = {'department': ('exact',), 'state': ('exact',), 'id': ('exact',)}
        excludes = ['addressbuildingname', 'complex_id', 'data_security', 'distribution_policy', 'fcode', 'foot_id',
                    'ftype', 'globalid', 'gnis_id', 'islandmark', 'loaddate', 'objectid', 'permanent_identifier',
                    'pointlocationtype', 'source_datadesc', 'source_datasetid', 'source_featureid',
                    'source_originator', 'admintype'
                    ]
        serializer = PrettyJSONSerializer()
        limit = 120

class StaffingResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The ResponseCapability API.
    """

    firestation = fields.ForeignKey(FireStationResource, 'firestation')

    class Meta:
        resource_name = 'staffing'
        queryset = Staffing.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='firestation.department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='change_firedepartment')
        authentication = MultiAuthentication(SessionAuthentication(), ApiKeyAuthentication())
        filtering = {'firestation': ALL}
        validation = FormValidation(form_class=StaffingForm)
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get', 'put', 'delete']
        serializer = PrettyJSONSerializer()
        always_return_data = True

class WeatherWarningResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Weather API mege with department id.
    """
    department = fields.ForeignKey(FireDepartmentResource, 'department', null=True)

    class Meta:
        resource_name = 'weather-warning'
        expired = timezone.now()
        # production when Task is running
        queryset = DepartmentWarnings.objects.filter(expiredate__gte=expired)
        # only show all warnings #good for testing Warnings
        # queryset = DepartmentWarnings.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='change_firedepartment')
        filtering = {'department': ('exact',), 'state': ('exact',), 'id': ('exact',)}
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get']
        cache = SimpleCache(timeout=120)
        serializer = PrettyJSONSerializer()
        always_return_data = True
<commit_msg>update to expiredate for weather warning filter<commit_after>import json
import logging
from .forms import StaffingForm
from .models import FireStation, Staffing, FireDepartment
from firecares.weather.models import DepartmentWarnings
from django.core.serializers.json import DjangoJSONEncoder
from django.contrib.gis import geos
from tastypie import fields
from tastypie.authentication import SessionAuthentication, ApiKeyAuthentication, MultiAuthentication, Authentication
from tastypie.authorization import DjangoAuthorization
from tastypie.cache import SimpleCache
from tastypie.constants import ALL
from tastypie.contrib.gis.resources import ModelResource
from tastypie.exceptions import Unauthorized, TastypieError
from tastypie.serializers import Serializer
from tastypie.validation import FormValidation
from guardian.core import ObjectPermissionChecker
import datetime

logger = logging.getLogger(__name__)

# Note this is from one of my other projects, not sure it is actually needed or not.
class SessionAuth(SessionAuthentication):
    """
    This is a hack to fix a bug which returns occasional TypeErrors returned from SessionAuthentication.

    About:
    Every now and then the super class' get_identifier returns a TypeError (getattr(): attribute name must be string).
    It seems that the logic that returns the string used for the username sometimes returns None.

    """
    def get_identifier(self, request):
        """
        Provides a unique string identifier for the requestor.

        This implementation returns the user's username.
        """
        try:
            return super(SessionAuth, self).get_identifier(request)
        except TypeError:
            return getattr(request.user, 'username')

class PrettyJSONSerializer(Serializer):
    json_indent = 2

    def to_json(self, data, options=None):
        options = options or {}
        data = self.to_simple(data, options)
        return json.dumps(data, cls=DjangoJSONEncoder,
                          sort_keys=True, ensure_ascii=False, indent=self.json_indent)

class JSONDefaultModelResourceMixin(object):
    def determine_format(self, request):
        return 'application/json' if not request.GET.get('format') else super(JSONDefaultModelResourceMixin, self).determine_format(request)

class PropertyResolutionFailedException(TastypieError):
    def __init__(self, property_path, class_name, *args, **kwargs):
        msg = '"{}" does not resolve for class type {}'.format(property_path, class_name)
        super(PropertyResolutionFailedException, self).__init__(msg, *args, **kwargs)

# Pulled/modified from https://gist.github.com/7wonders/6557760
class GuardianAuthorization(DjangoAuthorization):
    """
    :create_permission_code:
        the permission code that signifies the user can create one of these objects
    :view_permission_code:
        the permission code that signifies the user can view the detail
    :update_permission_code:
        the permission code that signifies the user can update one of these objects
    :remove_permission_code:
        the permission code that signifies the user can remove one of these objects
    :kwargs:
        other permission codes
        class Something(models.Model):
            name = models.CharField()
        class SomethingResource(ModelResource):
            class Meta:
                queryset = Something.objects.all()
                authorization = GuardianAuthorization(
                    delegate_to_property = None,  # property path (dot notation) to delegate authorization to
                    view_permission_code = 'can_view',  # empty view_permission_code allows anonymous users to view
                    create_permission_code = 'can_create',
                    update_permission_code = 'can_update',
                    delete_permission_code = 'can_delete'
                    )
    """

    def __init__(self, *args, **kwargs):
        # Allow for authorization to be delegated through a property on the object
        self.parent_property = kwargs.pop('delegate_to_property', None)
        self.view_permission_code = kwargs.pop("view_permission_code", 'can_view')
        self.create_permission_code = kwargs.pop("create_permission_code", 'can_create')
        self.update_permission_code = kwargs.pop("update_permission_code", 'can_update')
        self.delete_permission_code = kwargs.pop("delete_permission_code", 'can_delete')
        super(GuardianAuthorization, self).__init__(**kwargs)

    def resolve_property(self, obj, property_path):
        path = property_path.split('.')
        ret = obj
        for p in path:
            if hasattr(ret, p):
                ret = getattr(ret, p)
            else:
                raise PropertyResolutionFailedException(property_path, obj.__class__)
        return ret

    def generic_base_check(self, object_list, bundle):
        """
            Returns False if either:
                a) if the `object_list.model` doesn't have a `_meta` attribute
                b) the `bundle.request` object doesn have a `user` attribute
        """
        klass = self.base_checks(bundle.request, object_list.model)
        if klass is False:
            raise Unauthorized("You are not allowed to access that resource.")
        return True

    def generic_item_check(self, object_list, bundle, permission):
        if bundle.request.user.is_superuser:
            return True

        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")

        checker = ObjectPermissionChecker(bundle.request.user)
        if self.parent_property:
            obj = self.resolve_property(bundle.obj, self.parent_property)
        else:
            obj = bundle.obj

        # When the property resolves to None, prevent access by default
        if obj is None or not checker.has_perm(permission, obj):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    def generic_list_check(self, object_list, bundle, permission):
        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")
        user = bundle.request.user
        return [i for i in object_list if permission is None or user.has_perm(permission, i) or user.is_superuser]

    def generic_post_check(self, object_list, bundle, permission):
        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")
        user = bundle.request.user
        if not user.has_perm(permission):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    # List Checks
    def create_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.create_permission_code)

    def read_list(self, object_list, bundle):
        if not self.view_permission_code:
            return object_list

        return self.generic_list_check(object_list, bundle,
                                       self.view_permission_code)

    def update_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.update_permission_code)

    def delete_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.delete_permission_code)

    # Item Checks
    def create_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.create_permission_code)

    def read_detail(self, object_list, bundle):
        if not self.view_permission_code:
            return True
        return self.generic_item_check(object_list, bundle,
                                       self.view_permission_code)

    def update_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.update_permission_code)

    def delete_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.delete_permission_code)

class FireDepartmentResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Fire Department API.
    """

    class Meta:
        readonly_fields = ['owned_tracts_geom']
        resource_name = 'fire-departments'
        queryset = FireDepartment.objects.defer('owned_tracts_geom').filter(archived=False)
        authorization = GuardianAuthorization(view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='admin_firedepartment')
        authentication = MultiAuthentication(SessionAuthentication(), ApiKeyAuthentication())
        cache = SimpleCache()
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get', 'put']
        filtering = {'state': ALL, 'featured': ALL}
        serializer = PrettyJSONSerializer()
        limit = 120
        max_limit = 2000

    def __init__(self, **kwargs):
        super(FireDepartmentResource, self).__init__(**kwargs)
        for f in getattr(self.Meta, 'readonly_fields', []):
            self.fields[f].readonly = True

    def hydrate_geom(self, bundle):
        geom = bundle.data.get('geom')
        boundary = geos.GEOSGeometry(json.dumps(geom))
        if type(boundary) is geos.Polygon:
            bundle.data['geom'] = json.loads(geos.MultiPolygon(boundary).json)
        return bundle

    def dehydrate(self, bundle):
        """
        remove fields not requested

        note: this is called for every object, so works for both detail and
              list requests
        """
        only_fields = bundle.request.GET.get('fields')
        debug_fields = bundle.request.GET.get('fields_debug', False)
        if only_fields:
            only_fields = only_fields.split(',')
            fields_to_remove = [field for field in bundle.data.keys()
                                if field not in only_fields]
            for field in fields_to_remove:
                del bundle.data[field]
            if debug_fields:
                bundle.data['_fields_selected'] = [field for field in only_fields
                                                   if field in bundle.data.keys()]
                bundle.data['_fields_removed'] = fields_to_remove
        return bundle

class FireStationResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Fire Station API.
    """

    department = fields.ForeignKey(FireDepartmentResource, 'department', null=True)

    class Meta:
        resource_name = 'firestations'

        queryset = FireStation.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='admin_firedepartment')
        authentication = MultiAuthentication(Authentication(), SessionAuthentication(), ApiKeyAuthentication())
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get', 'put']
        filtering = {'department': ('exact',), 'state': ('exact',), 'id': ('exact',)}
        excludes = ['addressbuildingname', 'complex_id', 'data_security', 'distribution_policy', 'fcode', 'foot_id',
                    'ftype', 'globalid', 'gnis_id', 'islandmark', 'loaddate', 'objectid', 'permanent_identifier',
                    'pointlocationtype', 'source_datadesc', 'source_datasetid', 'source_featureid',
                    'source_originator', 'admintype'
                    ]
        serializer = PrettyJSONSerializer()
        limit = 120

class StaffingResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The ResponseCapability API.
    """

    firestation = fields.ForeignKey(FireStationResource, 'firestation')

    class Meta:
        resource_name = 'staffing'
        queryset = Staffing.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='firestation.department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='change_firedepartment')
        authentication = MultiAuthentication(SessionAuthentication(), ApiKeyAuthentication())
        filtering = {'firestation': ALL}
        validation = FormValidation(form_class=StaffingForm)
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get', 'put', 'delete']
        serializer = PrettyJSONSerializer()
        always_return_data = True

class WeatherWarningResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Weather API mege with department id.
    """
    department = fields.ForeignKey(FireDepartmentResource, 'department', null=True)

    class Meta:
        resource_name = 'weather-warning'
        queryset = DepartmentWarnings.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='change_firedepartment')
        filtering = {'department': ('exact',), 'state': ('exact',), 'id': ('exact',)}
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get']
        cache = SimpleCache(timeout=120)
        serializer = PrettyJSONSerializer()
        always_return_data = True

    def get_object_list(self, request):
        return super(WeatherWarningResource, self).get_object_list(request).filter(expiredate__gte=datetime.datetime.now)
<|endoftext|>


<commit_before>#coding: utf8
"""
units.py - Unit conversion module for Willie
Copyright © 2013, Elad Alfassa, <elad@fedoraproject.org>
Copyright © 2013, Dimitri Molenaars, <tyrope@tyrope.nl>
Licensed under the Eiffel Forum License 2.

"""
from __future__ import unicode_literals, division
from willie.module import commands, example, NOLIMIT
import re

find_temp = re.compile('(-?[0-9]*\.?[0-9]*)[ °]*(K|C|F)', re.IGNORECASE)
find_length = re.compile('([0-9]*\.?[0-9]*)[ ]*(mile[s]?|mi|inch|in|foot|feet|ft|yard[s]?|yd|(?:centi|kilo|)meter[s]?|[kc]?m)', re.IGNORECASE)

def f_to_c(temp):
    return (float(temp) - 32) * 5 / 9

def c_to_k(temp):
    return temp + 273.15

def c_to_f(temp):
    return (9.0 / 5.0 * temp + 32)

def k_to_c(temp):
    return temp - 273.15

@commands('temp')
@example('.temp 100F', '37.78°C = 100.00°F = 310.93K')
@example('.temp 100C', '100.00°C = 212.00°F = 373.15K')
@example('.temp 100K', '-173.15°C = -279.67°F = 100.00K')
def temperature(bot, trigger):
    """
    Convert temperatures
    """
    try:
        source = find_temp.match(trigger.group(2)).groups()
    except (AttributeError, TypeError):
        bot.reply("That's not a valid temperature.")
        return NOLIMIT
    unit = source[1].upper()
    numeric = float(source[0])
    celsius = 0
    if unit == 'C':
        celsius = numeric
    elif unit == 'F':
        celsius = f_to_c(numeric)
    elif unit == 'K':
        celsius = k_to_c(numeric)

    kelvin = c_to_k(celsius)
    fahrenheit = c_to_f(celsius)
    bot.reply("{:.2f}°C = {:.2f}°F = {:.2f}K".format(celsius, fahrenheit, kelvin))

@commands('length', 'distance')
@example('.distance 3m', '3.00m = 9 feet, 10.11 inches')
@example('.distance 3km', '3.00km = 1.86 miles')
@example('.distance 3 miles', '4.83km = 3.00 miles')
@example('.distance 3 inch', '7.62cm = 3.00 inches')
@example('.distance 3 feet', '91.44cm = 3 feet, 0.00 inches')
@example('.distance 3 yards', '2.74m = 9 feet, 0.00 inches')
def distance(bot, trigger):
    """
    Convert distances
    """
    try:
        source = find_length.match(trigger.group(2)).groups()
    except (AttributeError, TypeError):
        bot.reply("That's not a valid length unit.")
        return NOLIMIT
    unit = source[1].lower()
    numeric = float(source[0])
    meter = 0
    if unit in ("meters", "meter", "m"):
        meter = numeric
    elif unit in ("kilometers", "kilometer", "km"):
        meter = numeric * 1000
    elif unit in ("miles", "mile", "mi"):
        meter = numeric / 0.00062137
    elif unit in ("inch", "in"):
        meter = numeric / 39.370
    elif unit in ("centimeters", "centimeter", "cm"):
        meter = numeric // 100
    elif unit in ("feet", "foot", "ft"):
        meter = numeric / 3.2808
    elif unit in ("yards", "yard", "yd"):
        meter = numeric / (3.2808 / 3)

    if meter >= 1000:
        metric_part = '{:.2f}km'.format(meter / 1000)
    elif meter < 1:
        metric_part = '{:.2f}cm'.format(meter * 100)
    else:
        metric_part = '{:.2f}m'.format(meter)

    # Shit like this makes me hate being an American.
    inch = meter * 39.37
    foot = int(inch) // 12
    inch = inch - (foot * 12)
    yard = foot // 3
    mile = meter * 0.00062137

    if yard > 500:
        stupid_part = '{:.2f} miles'.format(mile)
    else:
        parts = []
        if yard >= 100:
            parts.append('{} yards'.format(yard))
            foot -= (yard * 3)

        if foot == 1:
            parts.append('1 foot')
        elif foot != 0:
            parts.append('{:.0f} feet'.format(foot))

        parts.append('{:.2f} inches'.format(inch))

        stupid_part = ', '.join(parts)

    bot.reply('{} = {}'.format(metric_part, stupid_part))

if __name__ == "__main__":
    from willie.test_tools import run_example_tests
    run_example_tests(__file__)
<commit_msg>[units] cm to meter should not be a floordiv, 150cm = 1.5m, and not 1m<commit_after>#coding: utf8
"""
units.py - Unit conversion module for Willie
Copyright © 2013, Elad Alfassa, <elad@fedoraproject.org>
Copyright © 2013, Dimitri Molenaars, <tyrope@tyrope.nl>
Licensed under the Eiffel Forum License 2.

"""
from __future__ import unicode_literals, division
from willie.module import commands, example, NOLIMIT
import re

find_temp = re.compile('(-?[0-9]*\.?[0-9]*)[ °]*(K|C|F)', re.IGNORECASE)
find_length = re.compile('([0-9]*\.?[0-9]*)[ ]*(mile[s]?|mi|inch|in|foot|feet|ft|yard[s]?|yd|(?:centi|kilo|)meter[s]?|[kc]?m)', re.IGNORECASE)

def f_to_c(temp):
    return (float(temp) - 32) * 5 / 9

def c_to_k(temp):
    return temp + 273.15

def c_to_f(temp):
    return (9.0 / 5.0 * temp + 32)

def k_to_c(temp):
    return temp - 273.15

@commands('temp')
@example('.temp 100F', '37.78°C = 100.00°F = 310.93K')
@example('.temp 100C', '100.00°C = 212.00°F = 373.15K')
@example('.temp 100K', '-173.15°C = -279.67°F = 100.00K')
def temperature(bot, trigger):
    """
    Convert temperatures
    """
    try:
        source = find_temp.match(trigger.group(2)).groups()
    except (AttributeError, TypeError):
        bot.reply("That's not a valid temperature.")
        return NOLIMIT
    unit = source[1].upper()
    numeric = float(source[0])
    celsius = 0
    if unit == 'C':
        celsius = numeric
    elif unit == 'F':
        celsius = f_to_c(numeric)
    elif unit == 'K':
        celsius = k_to_c(numeric)

    kelvin = c_to_k(celsius)
    fahrenheit = c_to_f(celsius)
    bot.reply("{:.2f}°C = {:.2f}°F = {:.2f}K".format(celsius, fahrenheit, kelvin))

@commands('length', 'distance')
@example('.distance 3m', '3.00m = 9 feet, 10.11 inches')
@example('.distance 3km', '3.00km = 1.86 miles')
@example('.distance 3 miles', '4.83km = 3.00 miles')
@example('.distance 3 inch', '7.62cm = 3.00 inches')
@example('.distance 3 feet', '91.44cm = 3 feet, 0.00 inches')
@example('.distance 3 yards', '2.74m = 9 feet, 0.00 inches')
def distance(bot, trigger):
    """
    Convert distances
    """
    try:
        source = find_length.match(trigger.group(2)).groups()
    except (AttributeError, TypeError):
        bot.reply("That's not a valid length unit.")
        return NOLIMIT
    unit = source[1].lower()
    numeric = float(source[0])
    meter = 0
    if unit in ("meters", "meter", "m"):
        meter = numeric
    elif unit in ("kilometers", "kilometer", "km"):
        meter = numeric * 1000
    elif unit in ("miles", "mile", "mi"):
        meter = numeric / 0.00062137
    elif unit in ("inch", "in"):
        meter = numeric / 39.370
    elif unit in ("centimeters", "centimeter", "cm"):
        meter = numeric / 100
    elif unit in ("feet", "foot", "ft"):
        meter = numeric / 3.2808
    elif unit in ("yards", "yard", "yd"):
        meter = numeric / (3.2808 / 3)

    if meter >= 1000:
        metric_part = '{:.2f}km'.format(meter / 1000)
    elif meter < 1:
        metric_part = '{:.2f}cm'.format(meter * 100)
    else:
        metric_part = '{:.2f}m'.format(meter)

    # Shit like this makes me hate being an American.
    inch = meter * 39.37
    foot = int(inch) // 12
    inch = inch - (foot * 12)
    yard = foot // 3
    mile = meter * 0.00062137

    if yard > 500:
        stupid_part = '{:.2f} miles'.format(mile)
    else:
        parts = []
        if yard >= 100:
            parts.append('{} yards'.format(yard))
            foot -= (yard * 3)

        if foot == 1:
            parts.append('1 foot')
        elif foot != 0:
            parts.append('{:.0f} feet'.format(foot))

        parts.append('{:.2f} inches'.format(inch))

        stupid_part = ', '.join(parts)

    bot.reply('{} = {}'.format(metric_part, stupid_part))

if __name__ == "__main__":
    from willie.test_tools import run_example_tests
    run_example_tests(__file__)
<|endoftext|>


<commit_before>import xml.sax.handler, xml.sax
import types
from toolkit import execute, warn

ALPINO = "/home/amcat/resources/Alpino"

class Pos:
    def __init__(self, main, major, minor):
        self.main = main
        self.major = major
        self.minor = minor

class Node:
    def __init__(self, begin, word, lemma, pos):
        self.word = word
        self.lemma = lemma
        self.pos = pos
        self.children = {}
        self.parents = {}
        self.begin = begin
    def addchild(self, child, rel):
        self.children[rel] = self.children.get(rel, set()) | set([child])
        child.parents[rel] = child.parents.get(rel, set()) | set([self])

    def prnt(self, indent="", rel="", blacklist=set()):
        if rel: rel = "(%s) " % rel
        print "%s%s%s (%s / %s / %s)" % (indent, rel, self.word,self.lemma, self.pos.major, self.pos.minor)
        if self in blacklist:
            print "%s  ..." % indent
        else:
            blacklist.add(self)
            for rel, child in self.getchildren(True):
                child.prnt(indent+"  ", rel,blacklist)

    def __str__(self):
        return "%s (%s / %s / %s)" % (self.word,self.lemma, self.pos.major, self.pos.minor)

    def getchildren(self, includerel = False):
        for rel, children in self.children.items():
            for child in children:
                if includerel:
                    yield rel, child
                else:
                    yield child

class Sentence:
    def __init__(self):
        self.nodeindex = {}

    def get(self, nodeid, *args, **kargs):
        return self.nodeindex.get(nodeid, None)

    def add(self, node, nodeid):
        if nodeid in self.nodeindex: raise Exception("Node %s already exists!" % nodeid)
        self.nodeindex[nodeid] = node

    def roots(self):
        roots = set()
        for node in self.nodeindex.values():
            if not node.parents:
                roots.add(node)
        return roots

    def dot(self, nodedefs = {}):
        rels, nodes = [], []
        style= 'graph [rankdir="TB",ranksep="0"];edge[dir="back",arrowsize=".4",fontsize="10",fontcolor="1,0,.4",fontname="arial"];node[fontname="arial",fontsize="9",height=".2",shape="box",color="1,0,1"]'
        def id(node): return "%s_%i" % (node.lemma, node.begin)
        for node in self.nodeindex.values():
            nstyle = {'label' : "%s\\n(%s)" % (node.lemma, node.pos.major)}
            nstyle.update(nodedefs.get(node.begin, {}))
            nodedef = ",".join('%s="%s"' % ab for ab in nstyle.items())
            nodes.append('"%s" [%s];' % (id(node), nodedef))
            for rel, child in node.getchildren(True):
                rels.append('"%s" -> "%s" [label="%s"];' % (id(node), id(child), rel))
        dot = "digraph G{\n%s\n\n%s\n\n%s\n}" % (style, "\n".join(nodes), "\n".join(rels))
        return dot

    def nodes(self):
        visited = set()

        stack = list(self.roots())
        while stack:
            node = stack.pop(0)
            if node in visited: continue
            visited.add(node)
            yield node
            stack += list(node.getchildren())

    def prnt(self):
        for root in self.roots():
            root.prnt()

def fromFullTriples(str):
    result = Sentence()
    def getnode(begin,word, lemma, pos, pos2, pos3):
        res = result.get(begin)
        if not res:
            if "(" in pos3:
                major, minor = pos3.split("(",1)
                minor = minor[:-1]
            else:
                major, minor = pos3,""
            pos = Pos(pos, major, minor)
            res = Node(begin, word, lemma, pos)
            result.add(res, begin)
        return res
    if _isString(str): str = str.split("\n")
    for line in str:
        if not line.strip(): continue
        fields = line.strip().split("|")
        if len(fields) <> 16: raise Exception("Cannot parse %r" % fields)
        plem, pword, pbegin, pend, ppos, ppos2, ppos3, rel, clem, cword, cbegin, cend, cpos, cpos2, cpos3,sid = fields
        pnode = getnode(pbegin, pword, plem, ppos, ppos2, ppos3)
        cnode = getnode(cbegin, cword, clem, cpos, cpos2, cpos3)
        r1, rel = rel.split("/")
        pnode.addchild(cnode, rel)

    return result

def tokenize(sent, alpinohome=ALPINO, errhandler=warn):
    cmd = "%s/Tokenization/tok" % alpinohome
    if not sent: return None
    out, err = execute(cmd, sent)
    if err and errhandler: errhandler(err)
    out = out.replace("\n"," ")
    return out

def parse(sent, alpinohome=ALPINO, errhandler=warn):
    cmd = "LD_LIBRARY_PATH=%s/create_bin ALPINO_HOME=%s %s/create_bin/Alpino demo=off end_hook=dependencies -parse" % (alpinohome, alpinohome, alpinohome)
    if not sent: return None
    #print cmd
    out, err = execute(cmd, sent)
    if err and errhandler: errhandler(err)
    #print out
    return out

def parseTriples(sent, alpinohome=ALPINO, errhandler=warn):
    out = parse(sent, alpinohome, errhandler)
    if not out: return None
    return fromFullTriples(out)

def fromdb(db, sid):
    sql = """select wordbegin, word, lemma, pos, major, minor
             from vw_parses_words_pos where sentenceid = %i""" % sid
    s = Sentence()
    for b,w,l,p,mj,mn in db.doQuery(sql):
        pos = Pos(p, mj,mn)
        node = Node(b,w,l, pos)
        s.add(node, b)

    sql = """select t.parentbegin, t.childbegin, r.name
             from parses_triples t inner join parses_rels r on t.relation = r.relid
             where sentenceid=%i """ % sid

    for p, c, rel in db.doQuery(sql):
        pn = s.get(p)
        cn = s.get(c)
        pn.addchild(cn, rel)

    return s

################## AUXILLIARY FUNCTIONS ####################

def _isString(obj):
    return type(obj) in types.StringTypes

################## TEST DRIVER METHOD ######################

if __name__ == '__main__':
    import sys, dbtoolkit
    triples = dbtoolkit.anokoDB().getValue("SELECT parse FROM sentences_parses WHERE parsejobid>9 AND sentenceid=%i" % int(sys.argv[1]))
    s = fromFullTriples(triples)
    s.prnt()
<commit_msg>[svn r513] bugfix tokenize<commit_after>import xml.sax.handler, xml.sax
import types
from toolkit import execute, warn

ALPINO = "/home/amcat/resources/Alpino"

class Pos:
    def __init__(self, main, major, minor):
        self.main = main
        self.major = major
        self.minor = minor

class Node:
    def __init__(self, begin, word, lemma, pos):
        self.word = word
        self.lemma = lemma
        self.pos = pos
        self.children = {}
        self.parents = {}
        self.begin = begin
    def addchild(self, child, rel):
        self.children[rel] = self.children.get(rel, set()) | set([child])
        child.parents[rel] = child.parents.get(rel, set()) | set([self])

    def prnt(self, indent="", rel="", blacklist=set()):
        if rel: rel = "(%s) " % rel
        print "%s%s%s (%s / %s / %s)" % (indent, rel, self.word,self.lemma, self.pos.major, self.pos.minor)
        if self in blacklist:
            print "%s  ..." % indent
        else:
            blacklist.add(self)
            for rel, child in self.getchildren(True):
                child.prnt(indent+"  ", rel,blacklist)

    def __str__(self):
        return "%s (%s / %s / %s)" % (self.word,self.lemma, self.pos.major, self.pos.minor)

    def getchildren(self, includerel = False):
        for rel, children in self.children.items():
            for child in children:
                if includerel:
                    yield rel, child
                else:
                    yield child

class Sentence:
    def __init__(self):
        self.nodeindex = {}

    def get(self, nodeid, *args, **kargs):
        return self.nodeindex.get(nodeid, None)

    def add(self, node, nodeid):
        if nodeid in self.nodeindex: raise Exception("Node %s already exists!" % nodeid)
        self.nodeindex[nodeid] = node

    def roots(self):
        roots = set()
        for node in self.nodeindex.values():
            if not node.parents:
                roots.add(node)
        return roots

    def dot(self, nodedefs = {}):
        rels, nodes = [], []
        style= 'graph [rankdir="TB",ranksep="0"];edge[dir="back",arrowsize=".4",fontsize="10",fontcolor="1,0,.4",fontname="arial"];node[fontname="arial",fontsize="9",height=".2",shape="box",color="1,0,1"]'
        def id(node): return "%s_%i" % (node.lemma, node.begin)
        for node in self.nodeindex.values():
            nstyle = {'label' : "%s\\n(%s)" % (node.lemma, node.pos.major)}
            nstyle.update(nodedefs.get(node.begin, {}))
            nodedef = ",".join('%s="%s"' % ab for ab in nstyle.items())
            nodes.append('"%s" [%s];' % (id(node), nodedef))
            for rel, child in node.getchildren(True):
                rels.append('"%s" -> "%s" [label="%s"];' % (id(node), id(child), rel))
        dot = "digraph G{\n%s\n\n%s\n\n%s\n}" % (style, "\n".join(nodes), "\n".join(rels))
        return dot

    def nodes(self):
        visited = set()

        stack = list(self.roots())
        while stack:
            node = stack.pop(0)
            if node in visited: continue
            visited.add(node)
            yield node
            stack += list(node.getchildren())

    def prnt(self):
        for root in self.roots():
            root.prnt()

def fromFullTriples(str):
    result = Sentence()
    def getnode(begin,word, lemma, pos, pos2, pos3):
        res = result.get(begin)
        if not res:
            if "(" in pos3:
                major, minor = pos3.split("(",1)
                minor = minor[:-1]
            else:
                major, minor = pos3,""
            pos = Pos(pos, major, minor)
            res = Node(begin, word, lemma, pos)
            result.add(res, begin)
        return res
    if _isString(str): str = str.split("\n")
    for line in str:
        if not line.strip(): continue
        fields = line.strip().split("|")
        if len(fields) <> 16: raise Exception("Cannot parse %r" % fields)
        plem, pword, pbegin, pend, ppos, ppos2, ppos3, rel, clem, cword, cbegin, cend, cpos, cpos2, cpos3,sid = fields
        pnode = getnode(pbegin, pword, plem, ppos, ppos2, ppos3)
        cnode = getnode(cbegin, cword, clem, cpos, cpos2, cpos3)
        r1, rel = rel.split("/")
        pnode.addchild(cnode, rel)

    return result

def tokenize(sent, alpinohome=ALPINO, errhandler=warn):
    cmd = "%s/Tokenization/tok" % alpinohome
    if not sent: return None
    out, err = execute(cmd, sent+" ")
    print "%r -> %r" % (sent, out)
    if err and errhandler: errhandler(err)
    out = out.replace("\n"," ")
    return out

def parse(sent, alpinohome=ALPINO, errhandler=warn):
    cmd = "LD_LIBRARY_PATH=%s/create_bin ALPINO_HOME=%s %s/create_bin/Alpino demo=off end_hook=dependencies -parse" % (alpinohome, alpinohome, alpinohome)
    if not sent: return None
    #print cmd
    out, err = execute(cmd, sent)
    if err and errhandler: errhandler(err)
    #print out
    return out

def parseTriples(sent, alpinohome=ALPINO, errhandler=warn):
    out = parse(sent, alpinohome, errhandler)
    if not out: return None
    return fromFullTriples(out)

def fromdb(db, sid):
    sql = """select wordbegin, word, lemma, pos, major, minor
             from vw_parses_words_pos where sentenceid = %i""" % sid
    s = Sentence()
    for b,w,l,p,mj,mn in db.doQuery(sql):
        pos = Pos(p, mj,mn)
        node = Node(b,w,l, pos)
        s.add(node, b)

    sql = """select t.parentbegin, t.childbegin, r.name
             from parses_triples t inner join parses_rels r on t.relation = r.relid
             where sentenceid=%i """ % sid

    for p, c, rel in db.doQuery(sql):
        pn = s.get(p)
        cn = s.get(c)
        pn.addchild(cn, rel)

    return s

################## AUXILLIARY FUNCTIONS ####################

def _isString(obj):
    return type(obj) in types.StringTypes

################## TEST DRIVER METHOD ######################

if __name__ == '__main__':
    import sys, dbtoolkit
    triples = dbtoolkit.anokoDB().getValue("SELECT parse FROM sentences_parses WHERE parsejobid>9 AND sentenceid=%i" % int(sys.argv[1]))
    s = fromFullTriples(triples)
    s.prnt()
<|endoftext|>


<commit_before>from dal import autocomplete
from django.core.validators import ValidationError
from django.db.models import Q, Count
from django.http import JsonResponse, Http404
from django.views.generic import ListView, DetailView
from django.shortcuts import get_object_or_404
from djiffy.models import Canvas
from SolrClient.exceptions import SolrError

from winthrop.books.models import Book, Publisher, Language, Subject
from winthrop.books.forms import SearchForm
from winthrop.common.solr import PagedSolrQuery
from winthrop.common.views import LastModifiedListMixin, LastModifiedMixin, \
    VaryOnHeadersMixin

class BookListView(ListView, LastModifiedListMixin, VaryOnHeadersMixin):
    model = Book
    template_name = 'books/book_list.html'
    paginate_by = 50
    form_class = SearchForm
    form = None
    vary_headers = ['X-Requested-With']

    def get_template_names(self):
        # when queried via ajax, return partial html for just the results section
        # (don't render the form or base template)
        if self.request.is_ajax():
            return 'books/snippets/book_list_results.html'
        return self.template_name

    def solr_query_opts(self):
        # NOTE: solr query logic used by both the view and to generate
        # last-modified value for header/conditional display

        form_opts = self.request.GET.copy()

        # NOTE: default sort logic borrowed from PPA
        # if relevance sort is requested but there is no keyword search
        # term present, clear it out and fallback to default sort
        if not 'query' in form_opts and 'sort' in form_opts and \
          form_opts['sort'] == 'relevance':
            del form_opts['sort']

        for key, val in self.form_class.defaults.items():
            # set as list to avoid nested lists
            # follows solution using in derrida-django for InstanceListView
            if isinstance(val, list):
                form_opts.setlistdefault(key, val)
            else:
                form_opts.setdefault(key, val)

        self.form = self.form_class(form_opts)
        # if the form is not valid, bail out
        if not self.form.is_valid():
            raise ValidationError('Search form is not valid')

        search_opts = {}
        if self.form.is_valid():
            search_opts = self.form.cleaned_data

        query = search_opts.get("query", None)

        solr_q = '*:*'
        # fields = '*'
        if query:
            solr_q = 'text:(%s)' % query
            # NOTE: score is needed in field list if we want to
            # display it for debugging solr indexing
            # fields = '*,score'

        ## faceting and filtering
        # list of filter queries; restrict to books, filter by
        # any facet fields that are specified
        # range logic adapted from PPA/Derrida projects
        range_opts = {
            'facet.range': self.form.range_facets
        }
        filter_qs = ['content_type:(%s)' % Book.content_type()]

        for range_facet in self.form.range_facets:
            # range filter requested in search options
            start = end = None
            # if start or end is specified on the form, add a filter query
            if range_facet in search_opts and search_opts[range_facet]:
                start, end = search_opts[range_facet].split('-')
                range_filter = '[%s TO %s]' % (start or '*', end or '*')
                # find works restricted by range
                filter_qs.append('%s:%s' % (range_facet, range_filter))

            # get minimum and maximum pub date values from the db
            pubmin, pubmax = self.form.pub_date_minmax()

            # NOTE: hard-coded values are fallback logic for when
            # no contents are in the database and pubmin/pubmax are None
            start = int(start) if start else pubmin or 0
            end = int(end) if end else pubmax or 1800

            # Configure range facet options specific to current field, to
            # support more than one range facet (even though not currently needed)
            range_opts.update({
                # current range filter
                'f.%s.facet.range.start' % range_facet: start,
                # NOTE: per facet.range.include documentation, default behavior
                # is to include lower bound and exclude upper bound.
                # For simplicity, increase range end by one.
                'f.%s.facet.range.end' % range_facet: end + 1,
                # calculate gap based start and end & desired number of slices
                # ideally, generate 24 slices; minimum gap size of 1
                'f.%s.facet.range.gap' % range_facet: max(1, int((end - start) / 24)),
                # restrict last range to *actual* maximum value
                'f.%s.facet.range.hardend' % range_facet: True
            })

        # check for facet filters that should be enabled
        for solr_field, form_field in self.form.solr_facet_fields.items():
            field_values = search_opts.get(form_field, None)
            if field_values:
                # Find matches for any of the terms.
                # Tag the filter using the form field name so that it
                # can be excluded when generating the facet, to get proper counts
                filter_qs.append('{!tag=%s}%s:(%s)' % \
                    (form_field, solr_field,
                     ' OR '.join('"%s"' % val for val in field_values)))

        # Construct list of facet fields to return.
        # Exclude tags based on form field name, so that counts will be
        # correct for multi-select faceting.
        # See https://lucene.apache.org/solr/guide/6_6/faceting.html#Faceting-TaggingandExcludingFilters
        # Use key to Set output label to form field name instead of solr field
        facet_fields = ['{!ex=%s key=%s}%s' % (form_field, form_field, solr_field)
                        for solr_field, form_field in self.form.solr_facet_fields.items()]

        ## sorting
        solr_sort = 'last_modified desc'
        sort = search_opts.get("sort", None)

        if sort:
            solr_sort = self.form.get_solr_sort_field(sort)

        solr_opts = {
            'q': solr_q,
            'sort': solr_sort,
            # 'fl': fields,
            # turn on faceting and add any self.form facet_fields
            'facet': 'true',
            'facet.field': facet_fields,
            'facet.limit': -1,
            # sort by alpha on facet label rather than count
            'facet.sort': 'index',
            'fq': filter_qs,
            # enabling highlighting for debugging search customization
            'hl': True,
            'hl.fl': 'text',
            'hl.snippets': 3,
            'hl.method': 'unified'
        }
        solr_opts.update(range_opts)
        return solr_opts

    def get_queryset(self, **kwargs):
        # return all books, filtering on content type
        try:
            return PagedSolrQuery(self.solr_query_opts())
        except ValidationError:
            # if the form is not valid, return an empty queryset and bail out
            # (queryset needed for django paginator)
            return Book.objects.none()

    def get_context_data(self, **kwargs):
        highlights = None
        try:
            # catch an error querying solr when the search terms cannot be parsed
            # (e.g., incomplete exact phrase)
            context = super().get_context_data(**kwargs)

            # populate form field choices based on facets
            # (may not actually be displayed except as a fallback and for testing)
            self.form.set_choices_from_facets(self.object_list.get_facets())

            # temporarily include highlights to test search index customization
            # retrieve inside try/except in case of syntax errro
            highlights = self.object_list.get_highlighting()

        except SolrError as solr_err:
            context = {'object_list': []}
            if 'Cannot parse' in str(solr_err):
                error_msg = 'Unable to parse search query; please revise and try again.'
            else:
                # NOTE: this error should possibly be raised; 500 error?
                error_msg = 'Something went wrong.'
            context['error'] = error_msg
        context.update({
            'search_form': self.form,
            # temporarily include highlights to test search index customization
            'highlights': highlights
        })
        return context

    def last_modified(self):
        '''override last modified logic to work with Solr'''
        try:
            query_opts = self.solr_query_opts()
        except ValidationError:
            return

        # override sort to return most recent modification date,
        # only return last modified value and nothing else
        query_opts.update({
            'sort': 'last_modified desc',
            'fl': 'last_modified'
        })

        # if a syntax or other solr error happens, no date to return
        try:
            psq = PagedSolrQuery(query_opts)
            # Solr stores date in isoformat; convert to datetime
            return self.solr_timestamp_to_datetime(psq[0]['last_modified'])
            # skip extra call to Solr to check count and just grab the first
            # item if it exists
        except (IndexError, SolrError):
            pass

class BookFacetJSONView(BookListView):
    error_code = None

    def get_context_data(self, **kwargs):
        # skip normal context handling and only return count and facets

        # get paginator for including number of pages, but skip other
        # context data logic needed for full result view
        paginator = self.get_paginator(self.object_list, self.paginate_by)
        try:
            return {
                'total': self.object_list.count(),
                'resultsPerPage': self.paginate_by,
                'pages': paginator.num_pages,
                'facets': self.object_list.get_facets(),
                'range_facets': self.object_list.get_facets_ranges()
            }
        except SolrError as solr_err:
            error_msg = 'Something went wrong.'
            self.error_code = 500
            if 'Cannot parse' in str(solr_err):
                error_msg = ('Unable to parse search query; '
                             'please revise and try again.')
                self.error_code = 400
            return {'error': error_msg}

    def render_to_response(self, context, **response_kwargs):
        response = JsonResponse(context, **response_kwargs)
        # if something went wrong, set an error code on the response before returning
        if 'error' in context and self.error_code:
            response.status_code = self.error_code
        return response

class BookDetailView(DetailView, LastModifiedMixin):
    model = Book

    def last_modified(self):
        '''book model doesn't track update; get last index modification from Solr instead'''

        # if there is a solr error, skip last-modified behavior and display page
        try:
            psq = PagedSolrQuery({
                'q': 'id:"%s"' % self.object.index_id(),
                'fl': 'last_modified'
            })
            if psq.count():
                # Solr stores date in isoformat; convert to datetime
                return self.solr_timestamp_to_datetime(psq[0]['last_modified'])
        except SolrError:
            pass

class BookPageView(ListView):
    model = Canvas
    template_name = 'books/book_page_thumbnails.html'
    context_object_name = 'pages'

    def get_queryset(self):
        qs = super().get_queryset()
        # filter canvas based on book slug, and annotate with annotation counts
        # so template can display indicators for text and graphic annotations
        return qs.filter(manifest__book__slug=self.kwargs['slug']) \
                 .annotate(textual_annotation=Count('annotation', filter=Q(annotation__text='')),
                           graphical_annotation=Count('annotation', filter=Q(annotation__text__ne=''))) \
                 .values('iiif_image_id', 'label', 'textual_annotation', 'graphical_annotation')

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        # add book to context for title display and link to main book page
        # 404 if book does not exist or does not have a digital editon
        book = get_object_or_404(Book, slug=self.kwargs['slug'])
        if not book.digital_edition:
            raise Http404

        context['book'] = book
        return context

class PublisherAutocomplete(autocomplete.Select2QuerySetView):
    '''Basic publisher autocomplete lookup, for use with
    django-autocomplete-light.  Restricted to staff only.'''
    # NOTE staff restrection applied in url config

    def get_queryset(self):
        return Publisher.objects.filter(name__icontains=self.q)

class CanvasAutocomplete(autocomplete.Select2QuerySetView):
    '''Canvas lookup for admin interface'''
    def get_queryset(self):
        return Canvas.objects.filter(
            Q(label__icontains=self.q) |
            Q(uri__contains=self.q)
        )

class LanguageAutocomplete(autocomplete.Select2QuerySetView):
    '''Autocomplete for languages in the controlled vocabulary list'''

    def get_queryset(self):
        return Language.objects.filter(name__icontains=self.q)

class SubjectAutocomplete(autocomplete.Select2QuerySetView):
    '''Basic autocomplete view for Subjects'''
    def get_queryset(self):
        return Subject.objects.filter(name__icontains=self.q)
<commit_msg>Remove pagination information from json facet data<commit_after>from dal import autocomplete
from django.core.validators import ValidationError
from django.db.models import Q, Count
from django.http import JsonResponse, Http404
from django.views.generic import ListView, DetailView
from django.shortcuts import get_object_or_404
from djiffy.models import Canvas
from SolrClient.exceptions import SolrError

from winthrop.books.models import Book, Publisher, Language, Subject
from winthrop.books.forms import SearchForm
from winthrop.common.solr import PagedSolrQuery
from winthrop.common.views import LastModifiedListMixin, LastModifiedMixin, \
    VaryOnHeadersMixin

class BookListView(ListView, LastModifiedListMixin, VaryOnHeadersMixin):
    model = Book
    template_name = 'books/book_list.html'
    paginate_by = 50
    form_class = SearchForm
    form = None
    vary_headers = ['X-Requested-With']

    def get_template_names(self):
        # when queried via ajax, return partial html for just the results section
        # (don't render the form or base template)
        if self.request.is_ajax():
            return 'books/snippets/book_list_results.html'
        return self.template_name

    def solr_query_opts(self):
        # NOTE: solr query logic used by both the view and to generate
        # last-modified value for header/conditional display

        form_opts = self.request.GET.copy()

        # NOTE: default sort logic borrowed from PPA
        # if relevance sort is requested but there is no keyword search
        # term present, clear it out and fallback to default sort
        if not 'query' in form_opts and 'sort' in form_opts and \
          form_opts['sort'] == 'relevance':
            del form_opts['sort']

        for key, val in self.form_class.defaults.items():
            # set as list to avoid nested lists
            # follows solution using in derrida-django for InstanceListView
            if isinstance(val, list):
                form_opts.setlistdefault(key, val)
            else:
                form_opts.setdefault(key, val)

        self.form = self.form_class(form_opts)
        # if the form is not valid, bail out
        if not self.form.is_valid():
            raise ValidationError('Search form is not valid')

        search_opts = {}
        if self.form.is_valid():
            search_opts = self.form.cleaned_data

        query = search_opts.get("query", None)

        solr_q = '*:*'
        # fields = '*'
        if query:
            solr_q = 'text:(%s)' % query
            # NOTE: score is needed in field list if we want to
            # display it for debugging solr indexing
            # fields = '*,score'

        ## faceting and filtering
        # list of filter queries; restrict to books, filter by
        # any facet fields that are specified
        # range logic adapted from PPA/Derrida projects
        range_opts = {
            'facet.range': self.form.range_facets
        }
        filter_qs = ['content_type:(%s)' % Book.content_type()]

        for range_facet in self.form.range_facets:
            # range filter requested in search options
            start = end = None
            # if start or end is specified on the form, add a filter query
            if range_facet in search_opts and search_opts[range_facet]:
                start, end = search_opts[range_facet].split('-')
                range_filter = '[%s TO %s]' % (start or '*', end or '*')
                # find works restricted by range
                filter_qs.append('%s:%s' % (range_facet, range_filter))

            # get minimum and maximum pub date values from the db
            pubmin, pubmax = self.form.pub_date_minmax()

            # NOTE: hard-coded values are fallback logic for when
            # no contents are in the database and pubmin/pubmax are None
            start = int(start) if start else pubmin or 0
            end = int(end) if end else pubmax or 1800

            # Configure range facet options specific to current field, to
            # support more than one range facet (even though not currently needed)
            range_opts.update({
                # current range filter
                'f.%s.facet.range.start' % range_facet: start,
                # NOTE: per facet.range.include documentation, default behavior
                # is to include lower bound and exclude upper bound.
                # For simplicity, increase range end by one.
                'f.%s.facet.range.end' % range_facet: end + 1,
                # calculate gap based start and end & desired number of slices
                # ideally, generate 24 slices; minimum gap size of 1
                'f.%s.facet.range.gap' % range_facet: max(1, int((end - start) / 24)),
                # restrict last range to *actual* maximum value
                'f.%s.facet.range.hardend' % range_facet: True
            })

        # check for facet filters that should be enabled
        for solr_field, form_field in self.form.solr_facet_fields.items():
            field_values = search_opts.get(form_field, None)
            if field_values:
                # Find matches for any of the terms.
                # Tag the filter using the form field name so that it
                # can be excluded when generating the facet, to get proper counts
                filter_qs.append('{!tag=%s}%s:(%s)' % \
                    (form_field, solr_field,
                     ' OR '.join('"%s"' % val for val in field_values)))

        # Construct list of facet fields to return.
        # Exclude tags based on form field name, so that counts will be
        # correct for multi-select faceting.
        # See https://lucene.apache.org/solr/guide/6_6/faceting.html#Faceting-TaggingandExcludingFilters
        # Use key to Set output label to form field name instead of solr field
        facet_fields = ['{!ex=%s key=%s}%s' % (form_field, form_field, solr_field)
                        for solr_field, form_field in self.form.solr_facet_fields.items()]

        ## sorting
        solr_sort = 'last_modified desc'
        sort = search_opts.get("sort", None)

        if sort:
            solr_sort = self.form.get_solr_sort_field(sort)

        solr_opts = {
            'q': solr_q,
            'sort': solr_sort,
            # 'fl': fields,
            # turn on faceting and add any self.form facet_fields
            'facet': 'true',
            'facet.field': facet_fields,
            'facet.limit': -1,
            # sort by alpha on facet label rather than count
            'facet.sort': 'index',
            'fq': filter_qs,
            # enabling highlighting for debugging search customization
            'hl': True,
            'hl.fl': 'text',
            'hl.snippets': 3,
            'hl.method': 'unified'
        }
        solr_opts.update(range_opts)
        return solr_opts

    def get_queryset(self, **kwargs):
        # return all books, filtering on content type
        try:
            return PagedSolrQuery(self.solr_query_opts())
        except ValidationError:
            # if the form is not valid, return an empty queryset and bail out
            # (queryset needed for django paginator)
            return Book.objects.none()

    def get_context_data(self, **kwargs):
        highlights = None
        try:
            # catch an error querying solr when the search terms cannot be parsed
            # (e.g., incomplete exact phrase)
            context = super().get_context_data(**kwargs)

            # populate form field choices based on facets
            # (may not actually be displayed except as a fallback and for testing)
            self.form.set_choices_from_facets(self.object_list.get_facets())

            # temporarily include highlights to test search index customization
            # retrieve inside try/except in case of syntax errro
            highlights = self.object_list.get_highlighting()

        except SolrError as solr_err:
            context = {'object_list': []}
            if 'Cannot parse' in str(solr_err):
                error_msg = 'Unable to parse search query; please revise and try again.'
            else:
                # NOTE: this error should possibly be raised; 500 error?
                error_msg = 'Something went wrong.'
            context['error'] = error_msg
        context.update({
            'search_form': self.form,
            # temporarily include highlights to test search index customization
            'highlights': highlights
        })
        return context

    def last_modified(self):
        '''override last modified logic to work with Solr'''
        try:
            query_opts = self.solr_query_opts()
        except ValidationError:
            return

        # override sort to return most recent modification date,
        # only return last modified value and nothing else
        query_opts.update({
            'sort': 'last_modified desc',
            'fl': 'last_modified'
        })

        # if a syntax or other solr error happens, no date to return
        try:
            psq = PagedSolrQuery(query_opts)
            # Solr stores date in isoformat; convert to datetime
            return self.solr_timestamp_to_datetime(psq[0]['last_modified'])
            # skip extra call to Solr to check count and just grab the first
            # item if it exists
        except (IndexError, SolrError):
            pass

class BookFacetJSONView(BookListView):
    error_code = None

    def get_context_data(self, **kwargs):
        # skip normal context handling and only return count and facets

        try:
            return {
                'total': self.object_list.count(),
                'facets': self.object_list.get_facets(),
                'range_facets': self.object_list.get_facets_ranges()
            }
        except SolrError as solr_err:
            error_msg = 'Something went wrong.'
            self.error_code = 500
            if 'Cannot parse' in str(solr_err):
                error_msg = ('Unable to parse search query; '
                             'please revise and try again.')
                self.error_code = 400
            return {'error': error_msg}

    def render_to_response(self, context, **response_kwargs):
        response = JsonResponse(context, **response_kwargs)
        # if something went wrong, set an error code on the response before returning
        if 'error' in context and self.error_code:
            response.status_code = self.error_code
        return response

class BookDetailView(DetailView, LastModifiedMixin):
    model = Book

    def last_modified(self):
        '''book model doesn't track update; get last index modification from Solr instead'''

        # if there is a solr error, skip last-modified behavior and display page
        try:
            psq = PagedSolrQuery({
                'q': 'id:"%s"' % self.object.index_id(),
                'fl': 'last_modified'
            })
            if psq.count():
                # Solr stores date in isoformat; convert to datetime
                return self.solr_timestamp_to_datetime(psq[0]['last_modified'])
        except SolrError:
            pass

class BookPageView(ListView):
    model = Canvas
    template_name = 'books/book_page_thumbnails.html'
    context_object_name = 'pages'

    def get_queryset(self):
        qs = super().get_queryset()
        # filter canvas based on book slug, and annotate with annotation counts
        # so template can display indicators for text and graphic annotations
        return qs.filter(manifest__book__slug=self.kwargs['slug']) \
                 .annotate(textual_annotation=Count('annotation', filter=Q(annotation__text='')),
                           graphical_annotation=Count('annotation', filter=Q(annotation__text__ne=''))) \
                 .values('iiif_image_id', 'label', 'textual_annotation', 'graphical_annotation')

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        # add book to context for title display and link to main book page
        # 404 if book does not exist or does not have a digital editon
        book = get_object_or_404(Book, slug=self.kwargs['slug'])
        if not book.digital_edition:
            raise Http404

        context['book'] = book
        return context

class PublisherAutocomplete(autocomplete.Select2QuerySetView):
    '''Basic publisher autocomplete lookup, for use with
    django-autocomplete-light.  Restricted to staff only.'''
    # NOTE staff restrection applied in url config

    def get_queryset(self):
        return Publisher.objects.filter(name__icontains=self.q)

class CanvasAutocomplete(autocomplete.Select2QuerySetView):
    '''Canvas lookup for admin interface'''
    def get_queryset(self):
        return Canvas.objects.filter(
            Q(label__icontains=self.q) |
            Q(uri__contains=self.q)
        )

class LanguageAutocomplete(autocomplete.Select2QuerySetView):
    '''Autocomplete for languages in the controlled vocabulary list'''

    def get_queryset(self):
        return Language.objects.filter(name__icontains=self.q)

class SubjectAutocomplete(autocomplete.Select2QuerySetView):
    '''Basic autocomplete view for Subjects'''
    def get_queryset(self):
        return Subject.objects.filter(name__icontains=self.q)
<|endoftext|>


<commit_before>from fabfile.fabutils import checkup_git_repo, flo, install_package
from fabfile.fabutils import install_packages, install_user_command, run, task

def users_bin_dir():
    '''Put custom commands at '~/bin/'.'''
    # circumvent circular import, cf. http://stackoverflow.com/a/18486863
    from fabfile.setup import pencil
    pencil() # used by ~/bin/ep2svg
    install_packages([
        'dia',
        'inkscape', # used by ~/bin/svg2pdf
        'xsltproc', # used by ~/bin/ep2svg
    ])
    commands = [
        'alldia2pdf',
        'allep2svg',
        'allsvg2pdf',
        'dia2pdf',
        'ep2svg',
        'svg2pdf'
    ]
    for command in commands:
        install_user_command(command)

@task
def latex():
    '''Install all packages and tools required to compile my latex documents.

    A complete latex installation "requires" the execution of the task
    'setup.users_bin_dir', too.
    '''
    users_bin_dir()
    # circumvent circular import, cf. http://stackoverflow.com/a/18486863
    from fabfile.setup import latex
    latex()
    checkup_git_repo('https://github.com/theno/haw-inf-thesis-template.git')
<commit_msg>Comments<commit_after>from fabfile.fabutils import checkup_git_repo, flo, install_package
from fabfile.fabutils import install_packages, install_user_command, run, task

def users_bin_dir():
    '''Put custom commands at '~/bin/'

    For the conversion of diagrams into the pdf format:
    * dia2pdf, ep2svg, svg2pdf
    * alldia2pdf, allep2svg, alldia2pdf
    '''
    # circumvent circular import, cf. http://stackoverflow.com/a/18486863
    from fabfile.setup import pencil
    pencil() # used by ~/bin/ep2svg
    install_packages([
        'dia',
        'inkscape', # used by ~/bin/svg2pdf
        'xsltproc', # used by ~/bin/ep2svg
    ])
    commands = [
        'alldia2pdf',
        'allep2svg',
        'allsvg2pdf',
        'dia2pdf',
        'ep2svg',
        'svg2pdf'
    ]
    for command in commands:
        install_user_command(command)

@task
def latex():
    '''Install all packages and tools required to compile my latex documents.

    * Install or update a lot of latex packages.
    * Install or update pencil, dia, inkscape, xsltproc for diagrams and
      images.
    * Install or update util commands for conversion of dia, ep, svg into pdf
      files.
    * Checkout or update a haw-thesis template git repository which uses all of
      the upper mentioned tools.
    '''
    users_bin_dir()
    # circumvent circular import, cf. http://stackoverflow.com/a/18486863
    from fabfile.setup import latex
    latex()
    checkup_git_repo('https://github.com/theno/haw-inf-thesis-template.git')
<|endoftext|>


<commit_before>import subprocess
import fileinput
import shutil
from skimage import io, util
import os
from PIL import Image

# saturation levels
colourRange = ['2', '2e-1', '2e-2', '2e-3', '2e-4']
colourName = ['0', '1', '2', '3', '4']
# max modes to generate
maxMode = 6
# open template kat file
originalKat = open('cavity.kat', 'r')

# loop over m, then n, then colour range
for m in range(maxMode):
    for n in range(maxMode):
        for c in range(5):

                # generate names
                current = "cavity"+str(m)+str(n)+colourName[c]
                generateFileName = current+".kat"

                # make and/or change directory
                if not os.path.exists('rawData'): os.mkdir('rawData')
                shutil.copy("cavity.kat", 'rawData/'+generateFileName)
                os.chdir('rawData')

                # generate new lines for .kat
                temLine = "tem laser "+str(m)+" "+str(n)+" 0.1 0.0" #replace lines
                colourLine = "set cbrange[0:"+colourRange[c]+"]"

                # replacing lines in .kat
                with fileinput.FileInput(generateFileName, inplace=True, backup='.bak') as file:
                        for line in file:
                            line = line.rstrip()
                            if "#replaceMode" in line:
                                line = line.replace("#replaceMode", temLine)
                            elif "#replaceColour" in line:
                                line = line.replace("#replaceColour", colourLine)
                            print(line)

                # run the new kat file
                subprocess.run(["../../finesse/./kat", generateFileName])

                # crop generated image
                # GNUplot generates a margin for some reason
                # assume square image
                image = Image.open(current+'.png')
                width = image.size[0]
                height = width
                offset = height-1
                cropped = image.crop((width-offset, height-offset, width, height))
                cropped.save(current+'.png')

                # cleaning files
                fileList = os.listdir()
                notImages = [x for x in fileList if not '.png' in x]
                for fileName in notImages: os.remove(fileName)

                #move back up directory
                os.chdir('..')

originalKat.close()
<commit_msg>Backported createData.py to Python 2, merged code with generateData.py<commit_after>import subprocess
import fileinput
import shutil
from skimage import io, util
import os
from PIL import Image
import tqdm
from keras.preprocessing.image import ImageDataGenerator, array_to_img, \
    img_to_array, load_img

'''
Simple python script to generate data from Finesse
First we run Finesse iterated over mode numbers, m, n
(Just pure modes, no superpositions)
Fix up the GNUplot raw image
Then use Keras preprocessing package to generate modified data
(rotated, skewed, zoomed)
Then we add noise (after Keras, otherwise wacky things happen)
Ready to feed into neural network
'''
FINESSEDIR = "../../finesse/./kat"
# saturation levels
colourRange = ['2', '2e-1', '2e-2', '2e-3', '2e-4']
colourName = ['0', '1', '2', '3', '4']
# max mode for n and m to generate
maxMode = 6
# open template kat file
originalKat = open('cavity.kat', 'r')

# Define data folders
dataFolder = 'rawData'
newDataFolder = 'newData'

# Image extension
ext = '.png'

# No. images to generate per raw image
imageNum = 20

# loop over m, then n, then colour range
for m in tqdm(range(maxMode)):
    for n in range(maxMode):
        for c in range(5):

                # generate names
                current = "cavity"+str(m)+str(n)+colourName[c]
                generateFileName = current+".kat"

                # make and/or change directory
                if not os.path.exists(dataFolder): os.mkdir(dataFolder)
                shutil.copy("cavity.kat", 'rawData/'+generateFileName)
                os.chdir('rawData')

                # generate new lines for .kat
                temLine = "tem laser "+str(m)+" "+str(n)+" 0.1 0.0" #replace lines
                colourLine = "set cbrange[0:"+colourRange[c]+"]"

                # replacing lines in .kat
                with fileinput.FileInput(generateFileName, inplace=True, backup='.bak') as file:
                        for line in file:
                            line = line.rstrip()
                            if "#replaceMode" in line:
                                line = line.replace("#replaceMode", temLine)
                            elif "#replaceColour" in line:
                                line = line.replace("#replaceColour", colourLine)
                            print(line)

                # run the new kat file
                subprocess.run([FINESSEDIR, generateFileName])

                # crop generated image
                # GNUplot generates a margin for some reason
                # assume square image
                image = Image.open(current+ext)
                width = image.size[0]
                height = width
                offset = height-1
                cropped = image.crop((width-offset, height-offset, width, height))
                cropped.save(current+ext)

                # cleaning files
                fileList = os.listdir()
                notImages = [x for x in fileList if not ext in x]
                for fileName in notImages: os.remove(fileName)

                #move back up directory
                os.chdir('..')

originalKat.close()

# Preprocessing
# Data generators, using Keras
trainDatagen = ImageDataGenerator(
        rotation_range=30,
        width_shift_range=0.2,
        height_shift_range=0.2,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True,
        fill_mode='nearest')
testDatagen = ImageDataGenerator(
        rotation_range=25,
        width_shift_range=0.1,
        height_shift_range=0.1,
        shear_range=0.3,
        zoom_range=0.1,
        horizontal_flip=True,
        fill_mode='nearest')
#
os.chdir(dataFolder)
files = os.listdir()
imageList = [x for x in files if '.png' in x]
os.chdir('..')

possibleModes = [str(m)+str(n) for m in range(maxMode) for n in range(maxMode)]

# Create file structure if it does not exist
if not os.path.exists(newDataFolder): os.mkdir(newDataFolder)

# Generators which read pictures in subfolders of training and validation,
# indefinitely generate batches of augmented cavity images

print('Preprocessing raw data set')
for mode in tqdm(possibleModes):
       for image in [x for x in files if 'cavity'+mode in x]:
        loaded = load_img('rawData/'+image)
        array = img_to_array(loaded)
        array = array.reshape((1,)+array.shape)
        i = 0
        for batch in trainDatagen.flow(array, batch_size=1, save_to_dir=newDataFolder,
                                       save_prefix=mode, save_format=ext):
            i += 1
            if i > imageNum: break
        print('generated '+image+' set')

# Adding noise
def noisy(img):
    noisy = util.random_noise(loaded, mode='gaussian', clip=True)
    return noisy

for image in tqdm(newDataFolder):
    loaded = io.imread(newDataFolder+'/'+image)
    io.imsave(image, noisy(image))
<|endoftext|>