<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function

import os
import time
import itertools

import numpy as np
import tensorflow as tf
from six.moves import range

from niftynet.engine.input_buffer import TrainEvalInputBuffer
from niftynet.utilities import misc_common as util
from niftynet.utilities.input_placeholders import ImagePatch
import niftynet.engine.logging
from niftynet.application.common import ApplicationFactory

np.random.seed(seed=int(time.time()))

def run(net_class, param, volume_loader, device_str):
    application=ApplicationFactory(param)(net_class, param, volume_loader)
    # construct graph
    graph = tf.Graph()
    with graph.as_default(), tf.device('/cpu:0'):
        # defines a training element
        sampler = application.sampler()
        # construct train queue
        with tf.name_scope('DataQueue'):
            train_batch_runner = TrainEvalInputBuffer(
                batch_size=param.batch_size,
                capacity=max(param.queue_length, param.batch_size),
                sampler=sampler,
                shuffle=True)
        tower_grads = []
        train_dict = train_batch_runner.pop_batch_op
        # Scalar summaries for the console are averaged over GPU runs
        console_outputs=graph.get_collection_ref(niftynet.engine.logging.CONSOLE)
        console_outputs_cache=console_outputs[:]
        del console_outputs[:]
        tower_console_outputs=[]

        for i in range(0, max(param.num_gpus, 1)):
            with tf.device("/{}:{}".format(device_str, i)):
                grads = application.train(train_dict)
                tower_grads.append(grads)
                # note: only use batch stats from one GPU for batch_norm
                if i == 0:
                    bn_updates = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

                # record and clear summaries
                console_outputs=graph.get_collection_ref(niftynet.engine.logging.CONSOLE)
                tower_console_outputs.append(console_outputs[:])
                del console_outputs[:]
        # group gradient ops by op_type, not GPU
        tower_grads=list(zip(*tower_grads))
        # add apply_grad_op for each type of optimizer_op
        apply_grad_ops=[]
        with tf.name_scope('AccumulateGradients'):
          for idx,ops in enumerate(tower_grads):
            ave_grads = util.average_grads(ops)
            apply_grad_ops.append(application.optimizer.apply_gradients(ave_grads))

        # Add averaged summaries
        console_outputs=graph.get_collection_ref(niftynet.engine.logging.CONSOLE)
        console_outputs+=console_outputs_cache
        if len(tower_console_outputs)>1:
            # Averages are in name_scope for Tensorboard naming; summaries are outside for console naming
            with tf.name_scope('AccumulateConsoleLogs'):
                averaged_summaries=[]
                for replicated_output in zip(*tower_console_outputs):
                    averaged_summaries.append([replicated_output[0].op.name+'_avg',tf.reduce_mean([o.op.inputs[1] for o in replicated_output])])
            for tag,avg in averaged_summaries:
                tf.summary.scalar(tag, avg,[niftynet.engine.logging.CONSOLE,niftynet.engine.logging.LOG])
        else:
            console_outputs+=tower_console_outputs[0]
        # Track the moving averages of all trainable variables.
        with tf.name_scope('MovingAverages'):
            variable_averages = tf.train.ExponentialMovingAverage(0.9)
            var_averages_op = variable_averages.apply(tf.trainable_variables())
            # batch norm variables moving mean and var
            batchnorm_updates_op = tf.group(*bn_updates)
        # primary operations
        init_op = tf.global_variables_initializer()
        train_ops = [tf.group(apply_grad_op,
                            var_averages_op,
                            batchnorm_updates_op) for apply_grad_op in apply_grad_ops]
        logged_summaries = list(set([s for c in [niftynet.engine.logging.LOG,niftynet.engine.logging.CONSOLE] for s in tf.get_collection(c)]))
        write_summary_op = tf.summary.merge(logged_summaries)
        # saver
        variables_to_restore = variable_averages.variables_to_restore()
        saver = tf.train.Saver(max_to_keep=param.max_checkpoints, var_list=variables_to_restore)
        tf.Graph.finalize(graph)
    # run session
    config = tf.ConfigProto()
    config.log_device_placement = False
    config.allow_soft_placement = True
    # config.gpu_options.allow_growth = True
    start_time = time.time()
    with tf.Session(config=config, graph=graph) as sess:
        # prepare output directory
        if not os.path.exists(os.path.join(param.model_dir, 'models')):
            os.makedirs(os.path.join(param.model_dir, 'models'))
        root_dir = os.path.abspath(param.model_dir)
        # start or load session
        ckpt_name = os.path.join(root_dir, 'models', 'model.ckpt')
        if param.starting_iter > 0:
            model_str = '{}-{}'.format(ckpt_name, param.starting_iter)
            saver.restore(sess, model_str)
            print('Loading from {}...'.format(model_str))
        else:
            sess.run(init_op)
            print('Weights from random initialisations...')
        coord = tf.train.Coordinator()
        writer = tf.summary.FileWriter(os.path.join(root_dir, 'logs'),
                                       sess.graph)
        try:
            print('Filling the queue (this can take a few minutes)')
            train_batch_runner.run_threads(sess, coord, param.num_threads)
            # Flush the train_ops up to training iter (for applications with complex training protocols)
            train_op_sequence = application.train_op_generator(train_ops)
            _=itertools.islice(train_op_sequence,param.starting_iter)
            for i in range(param.max_iter - param.starting_iter):
                local_time = time.time()
                if coord.should_stop():
                    break
                current_iter = i + param.starting_iter
                train_op_list = train_op_sequence.__next__()
                for idx,train_op in enumerate(train_op_list):
                    ops_to_run=[train_op]
                    if idx==0:
                        console_summaries=tf.get_collection(niftynet.engine.logging.CONSOLE)
                        ops_to_run += console_summaries
                        if (current_iter % 20) == 0:
                            ops_to_run += [write_summary_op]
                    values = sess.run(ops_to_run)[1:]
                    if idx==0 and (current_iter % 20) == 0:
                        writer.add_summary(values.pop(), current_iter)
                    summary_string = ''.join([niftynet.engine.logging.console_summary_string(v) for v in values])
                    iter_time = time.time() - local_time
                    iter_str = '{:d}'.format(current_iter) if len(train_op_list)==1 else '{:d}.{:d}'.format(current_iter, idx)
                    print(('iter {}{}, ({:.3f}s)').format(
                        iter_str, summary_string, iter_time))
                if (current_iter % param.save_every_n) == 0 and i > 0:
                    saver.save(sess, ckpt_name, global_step=current_iter)
                    print('Iter {} model saved at {}'.format(
                        current_iter, ckpt_name))
        except KeyboardInterrupt:
            print('User cancelled training')
        except tf.errors.OutOfRangeError as e:
            pass
        except Exception:
            import sys, traceback
            exc_type, exc_value, exc_traceback = sys.exc_info()
            traceback.print_exception(
                exc_type, exc_value, exc_traceback, file=sys.stdout)
        finally:
            saver.save(sess, ckpt_name, global_step=param.max_iter)
            print('Last iteration model saved at {}'.format(ckpt_name))
            print('training.py (time in second) {:.2f}'.format(
                time.time() - start_time))
            train_batch_runner.close_all()
<commit_msg>Save all variables<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function

import os
import time
import itertools

import numpy as np
import tensorflow as tf
from six.moves import range

from niftynet.engine.input_buffer import TrainEvalInputBuffer
from niftynet.utilities import misc_common as util
from niftynet.utilities.input_placeholders import ImagePatch
import niftynet.engine.logging
from niftynet.application.common import ApplicationFactory

np.random.seed(seed=int(time.time()))

def run(net_class, param, volume_loader, device_str):
    application=ApplicationFactory(param)(net_class, param, volume_loader)
    # construct graph
    graph = tf.Graph()
    with graph.as_default(), tf.device('/cpu:0'):
        # defines a training element
        sampler = application.sampler()
        # construct train queue
        with tf.name_scope('DataQueue'):
            train_batch_runner = TrainEvalInputBuffer(
                batch_size=param.batch_size,
                capacity=max(param.queue_length, param.batch_size),
                sampler=sampler,
                shuffle=True)
        tower_grads = []
        train_dict = train_batch_runner.pop_batch_op
        # Scalar summaries for the console are averaged over GPU runs
        console_outputs=graph.get_collection_ref(niftynet.engine.logging.CONSOLE)
        console_outputs_cache=console_outputs[:]
        del console_outputs[:]
        tower_console_outputs=[]

        for i in range(0, max(param.num_gpus, 1)):
            with tf.device("/{}:{}".format(device_str, i)):
                grads = application.train(train_dict)
                tower_grads.append(grads)
                # note: only use batch stats from one GPU for batch_norm
                if i == 0:
                    bn_updates = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

                # record and clear summaries
                console_outputs=graph.get_collection_ref(niftynet.engine.logging.CONSOLE)
                tower_console_outputs.append(console_outputs[:])
                del console_outputs[:]
        # group gradient ops by op_type, not GPU
        tower_grads=list(zip(*tower_grads))
        # add apply_grad_op for each type of optimizer_op
        apply_grad_ops=[]
        with tf.name_scope('AccumulateGradients'):
          for idx,ops in enumerate(tower_grads):
            ave_grads = util.average_grads(ops)
            apply_grad_ops.append(application.optimizer.apply_gradients(ave_grads))

        # Add averaged summaries
        console_outputs=graph.get_collection_ref(niftynet.engine.logging.CONSOLE)
        console_outputs+=console_outputs_cache
        if len(tower_console_outputs)>1:
            # Averages are in name_scope for Tensorboard naming; summaries are outside for console naming
            with tf.name_scope('AccumulateConsoleLogs'):
                averaged_summaries=[]
                for replicated_output in zip(*tower_console_outputs):
                    averaged_summaries.append([replicated_output[0].op.name+'_avg',tf.reduce_mean([o.op.inputs[1] for o in replicated_output])])
            for tag,avg in averaged_summaries:
                tf.summary.scalar(tag, avg,[niftynet.engine.logging.CONSOLE,niftynet.engine.logging.LOG])
        else:
            console_outputs+=tower_console_outputs[0]
        # Track the moving averages of all trainable variables.
        with tf.name_scope('MovingAverages'):
            variable_averages = tf.train.ExponentialMovingAverage(0.9)
            var_averages_op = variable_averages.apply(tf.trainable_variables())
            # batch norm variables moving mean and var
            batchnorm_updates_op = tf.group(*bn_updates)
        # primary operations
        init_op = tf.global_variables_initializer()
        train_ops = [tf.group(apply_grad_op,
                            var_averages_op,
                            batchnorm_updates_op) for apply_grad_op in apply_grad_ops]
        logged_summaries = list(set([s for c in [niftynet.engine.logging.LOG,niftynet.engine.logging.CONSOLE] for s in tf.get_collection(c)]))
        write_summary_op = tf.summary.merge(logged_summaries)
        # saver
        variables_to_restore = variable_averages.variables_to_restore()
        saver = tf.train.Saver(max_to_keep=param.max_checkpoints)
        tf.Graph.finalize(graph)
    # run session
    config = tf.ConfigProto()
    config.log_device_placement = False
    config.allow_soft_placement = True
    # config.gpu_options.allow_growth = True
    start_time = time.time()
    with tf.Session(config=config, graph=graph) as sess:
        # prepare output directory
        if not os.path.exists(os.path.join(param.model_dir, 'models')):
            os.makedirs(os.path.join(param.model_dir, 'models'))
        root_dir = os.path.abspath(param.model_dir)
        # start or load session
        ckpt_name = os.path.join(root_dir, 'models', 'model.ckpt')
        if param.starting_iter > 0:
            model_str = '{}-{}'.format(ckpt_name, param.starting_iter)
            saver.restore(sess, model_str)
            print('Loading from {}...'.format(model_str))
        else:
            sess.run(init_op)
            print('Weights from random initialisations...')
        coord = tf.train.Coordinator()
        writer = tf.summary.FileWriter(os.path.join(root_dir, 'logs'),
                                       sess.graph)
        try:
            print('Filling the queue (this can take a few minutes)')
            train_batch_runner.run_threads(sess, coord, param.num_threads)
            # Flush the train_ops up to training iter (for applications with complex training protocols)
            train_op_sequence = application.train_op_generator(train_ops)
            _=itertools.islice(train_op_sequence,param.starting_iter)
            for i in range(param.max_iter - param.starting_iter):
                local_time = time.time()
                if coord.should_stop():
                    break
                current_iter = i + param.starting_iter
                train_op_list = train_op_sequence.__next__()
                for idx,train_op in enumerate(train_op_list):
                    ops_to_run=[train_op]
                    if idx==0:
                        console_summaries=tf.get_collection(niftynet.engine.logging.CONSOLE)
                        ops_to_run += console_summaries
                        if (current_iter % 20) == 0:
                            ops_to_run += [write_summary_op]
                    values = sess.run(ops_to_run)[1:]
                    if idx==0 and (current_iter % 20) == 0:
                        writer.add_summary(values.pop(), current_iter)
                    summary_string = ''.join([niftynet.engine.logging.console_summary_string(v) for v in values])
                    iter_time = time.time() - local_time
                    iter_str = '{:d}'.format(current_iter) if len(train_op_list)==1 else '{:d}.{:d}'.format(current_iter, idx)
                    print(('iter {}{}, ({:.3f}s)').format(
                        iter_str, summary_string, iter_time))
                if (current_iter % param.save_every_n) == 0 and i > 0:
                    saver.save(sess, ckpt_name, global_step=current_iter)
                    print('Iter {} model saved at {}'.format(
                        current_iter, ckpt_name))
        except KeyboardInterrupt:
            print('User cancelled training')
        except tf.errors.OutOfRangeError as e:
            pass
        except Exception:
            import sys, traceback
            exc_type, exc_value, exc_traceback = sys.exc_info()
            traceback.print_exception(
                exc_type, exc_value, exc_traceback, file=sys.stdout)
        finally:
            saver.save(sess, ckpt_name, global_step=param.max_iter)
            print('Last iteration model saved at {}'.format(ckpt_name))
            print('training.py (time in second) {:.2f}'.format(
                time.time() - start_time))
            train_batch_runner.close_all()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'Shinichi Nakagawa'

"""
選手の情報に関する振る舞いを提供するService
"""

import locale
from datetime import datetime

from service.const import POSITION_PITCHER, POSITION_BATTER
from service.stats import Stats as BaseballStats
from noball.settings import APPLICATION_NAME
from mlb.service.sean_lahman import SeanLahmanDB as MlbDB

class Stats(object):

    # サブドメイン名
    SUB_DOMAIN = 'mlb'
    # Queryのキー名
    QUERY_KEY = 'query_word'

    # 右/左/両の略称
    SHORT_NAME_RIGHT = 'R'
    SHORT_NAME_LEFT = 'L'
    SHORT_NAME_SWITCH = 'S'

    # 通貨記号（アメリカなので＄）
    CURRENCY = '$'
    # SALARYの分母(million)
    BASE_SALARY = 1000000

    # ピタゴラス勝率のべき乗
    PYTHAGORIAN_POWER = 1.8

    # 戻り値の件数(固定)
    STATS_LIMIT = 4

    def __init__(self, encode):
        locale.setlocale(locale.LC_NUMERIC, encode)

    def get_base_context(self):
        """
        戻り値定数セット
        :return: 戻り値(固定)
        """
        return {
            'APPLICATION_NAME': APPLICATION_NAME,
            'QUERY_KEY': Stats.QUERY_KEY,
            'SUB_DOMAIN': Stats.SUB_DOMAIN,
            'PLAYER_BATTER': POSITION_BATTER,
            'PLAYER_PITCHER': POSITION_PITCHER,
            'MENU_ENABLE': True,
        }

    def get_pythagoras_dataset(self, teams):
        """
        ピタゴラス勝率を含んだデータセットを返す
        :param teams: Team Model List
        :return: pythagoras % list
        """
        datasets = []
        for t in teams:
            d = {
                'team': t.teamid,
                'name': t.name,
                'div': t.divid,
                'rank': t.rank,
                'g': t.g,
                'w': t.w,
                'l': t.l,
                'r': t.r,
                'er': t.er,
                'win_p': Stats.calc_winning_percentage(t.w, t.g),
                'pyt_p': Stats.calc_pythagorean_expectation(t.r, t.ra)
            }
            d['pyt_p_w'] = round(t.g * d['pyt_p'], 0)
            d['pyt_p_l'] = t.g - d['pyt_p_w']
            datasets.append(d)
        return datasets

    def get_sabr_value_batter(self, player, stats, salary):
        """
        SABR Stats for batter
        :param player: Player master model
        :param stats: Player stats model
        :param salary: Player salary model
        :return: (dict)sabar stats data
        """
        # SABR Data(batter)
        _datasets = {
            'rc_list': [],
            'babip_list': [],
            'dunn_list': []
        }

        for batting in stats:
            year = batting.yearid
            pa = BaseballStats.pa(batting.ab, batting.bb, batting.hbp, batting.sf, batting.sh)
            single = BaseballStats.single(batting.h, batting.hr, batting.number_2b, batting.number_3b)
            rc = BaseballStats.rc(
                batting.h,
                batting.bb,
                batting.hbp,
                batting.cs,
                batting.gidp,
                batting.sf,
                batting.sh,
                batting.sb,
                batting.so,
                batting.ab,
                batting.ibb,
                single,
                batting.number_2b,
                batting.number_3b,
                batting.hr
            )

            _datasets['rc_list'].append(
                {
                    'year': year,
                    'rc': BaseballStats.rc27(rc, batting.ab, batting.h, batting.sh, batting.sf, batting.cs, batting.gidp)
                }
            )

            _datasets['dunn_list'].append(
                {
                    'year': year,
                    'dunn': BaseballStats.adam_dunn_batter(batting.hr, batting.bb, batting.so, pa)
                }
            )
            _datasets['babip_list'].append(
                {
                    'year': year,
                    'avg': BaseballStats.avg(batting.h, batting.ab),
                    'babip': BaseballStats.babip(batting.h, batting.hr, batting.ab, batting.so, batting.sf)
                }
            )
        return _datasets

    def get_sabr_value_pitcher(self, player, stats, salary):
        """
        SABR Stats for batter
        :param player: Player master model
        :param stats: Player stats model
        :param salary: Player salary model
        :return: (dict)sabar stats data
        """
        # SABR Data(pitcher)
        _datasets = {
            'whip_list': [],
            'hr9_list': [],
            'dunn_list': []
        }

        for pitching in stats:
            year = pitching.yearid

            ip = BaseballStats.ip(pitching.ipouts)

            _datasets['whip_list'].append(
                {
                    'year': year,
                    'whip': BaseballStats.whip(pitching.bb, pitching.h, ip)
                }
            )
            _datasets['hr9_list'].append(
                {
                    'year': year,
                    'hr9': BaseballStats.hr9(pitching.hr, ip),
                    'bb9': BaseballStats.bb9(pitching.bb, ip),
                    'so9': BaseballStats.so9(pitching.so, ip)
                }
            )
            _datasets['dunn_list'].append(
                {
                    'year': year,
                    'dunn': BaseballStats.adam_dunn_pitcher(pitching.hr, pitching.bb, pitching.hbp, pitching.so, pitching.bfp)
                }
            )
        return _datasets

    def get_home_value_pitcher(self, player, stats, salary):
        """
        Pitcher profile
        :param player: Player master model
        :param stats: Player stats model
        :param salary: Player salary model
        :return: pitcher profile(dict)
        """
        # statsから必要な値を修得
        w, l, so, er, ipo, year, teams = 0, 0, 0, 0, 0, 0, []
        if len(stats) > 0:
            pitching = stats[0]
            year = pitching.yearid
            w = pitching.w
            l = pitching.l
            so = pitching.so
            er = pitching.er
            ipo = pitching.ipouts
            teams.append(MlbDB.get_last_team(MlbDB.get_pitching(), year, pitching.playerid))

        # ipを計算
        ip = BaseballStats.ip(ipo)

        _prof = self._get_base_profile(player, year, teams, salary)
        _prof['position'] = POSITION_PITCHER.upper()
        _prof['win'] = w
        _prof['lose'] = l
        _prof['era'] = BaseballStats.era(er, ip)
        _prof['so'] = so
        return _prof

    def get_home_value_batter(self, player, stats, salary):
        """
        Batter profile
        :param player: Player master model
        :param stats: Player stats model
        :param salary: Player salary model
        :return: batter profile(dict)
        """
        # statsから必要な値を修得
        hr, rbi, h, ab, year, teams = 0, 0, 0, 0, 0, []
        if len(stats) > 0:
            batter = stats[0]
            year = batter.yearid
            h = batter.h
            ab = batter.ab
            hr = batter.hr
            rbi = rbi + batter.rbi
            teams.append(MlbDB.get_last_team(MlbDB.get_batting(), year, batter.playerid))

        _prof = self._get_base_profile(player, year, teams, salary)
        _prof['position'] = POSITION_BATTER.upper()
        _prof['avg'] = BaseballStats.avg(h, ab)
        _prof['hr'] = hr
        _prof['rbi'] = rbi
        return _prof

    def _get_base_profile(self, player, year, teams, salary):
        """
        Profile(Base)
        :param player: Player master model
        :param year: Last play year
        :param teams: teams
        :param salary: Player salary model
        :return: profile(dict)
        """
        return {
            'year': year,
            'team': ','.join(teams),
            'age': Stats.calc_age(player.birthyear),
            'birthday': '%d/%d/%d' % (player.birthyear, player.birthmonth, player.birthday),
            'salary': Stats._calc_salary(salary),
            'country': player.birthcountry,
            'city': player.birthcity,
            'bats': player.bats,
            'throws': player.throws,
            'display_name': ' '.join([player.namefirst, player.namelast])

        }

    @classmethod
    def _calc_salary(cls, salary):
        """
        年俸計算
        :param salary: salary model list
        :return:
        """
        sal = 0
        if len(salary) > 0:
            for s in salary:
                sal = sal + s.salary
        return Stats._get_salary(sal)

    @classmethod
    def _get_salary(cls, money):
        """
        表示用のsalary
        :param money: salary
        :return: (str) salary
        """
        # SALARY編集
        if money == None or money == '':
            return '?'
        _money = int(money)
        if _money >= Stats.BASE_SALARY:
            return '%s%sM' % (Stats.CURRENCY, "{:n}".format(_money / Stats.BASE_SALARY))
        else:
            return '%s%s' % (Stats.CURRENCY, "{:n}".format(_money))

    @classmethod
    def calc_age(cls, year):
        """
        calculate age
        :param year: (int)birth year
        :return: (int) age
        """
        now = datetime.now()
        return now.year - year

    @classmethod
    def calc_winning_percentage(cls, w, g):
        """
        勝率
        :param w: win
        :param g: games
        :return: winning percentage
        """
        return round(w / g, 3)

    @classmethod
    def calc_pythagorean_expectation(cls, r, ra):
        """
        ピタゴラス勝率
        :param r: Runs Scored
        :param ra: Runs Allowed
        :return: pythagorean expectation
        """
        r_power = r ** Stats.PYTHAGORIAN_POWER
        ra_power = ra ** Stats.PYTHAGORIAN_POWER
        return round(r_power / (r_power + ra_power), 3)

    @classmethod
    def exists_player(cls, firstname, lastname):
        """
        選手がいるかいないか
        :param firstname: Player's First name
        :param lastname: Player's Last name
        :return: True(exists) or False(not exists)
        """
        if MlbDB.player().filter(namefirst=firstname).filter(namelast=lastname).count() == 1:
            return True
        else:
            return False<commit_msg>Salaryが全額加算されるバグを修正<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'Shinichi Nakagawa'

"""
選手の情報に関する振る舞いを提供するService
"""

import locale
from datetime import datetime

from service.const import POSITION_PITCHER, POSITION_BATTER
from service.stats import Stats as BaseballStats
from noball.settings import APPLICATION_NAME
from mlb.service.sean_lahman import SeanLahmanDB as MlbDB

class Stats(object):

    # サブドメイン名
    SUB_DOMAIN = 'mlb'
    # Queryのキー名
    QUERY_KEY = 'query_word'

    # 右/左/両の略称
    SHORT_NAME_RIGHT = 'R'
    SHORT_NAME_LEFT = 'L'
    SHORT_NAME_SWITCH = 'S'

    # 通貨記号（アメリカなので＄）
    CURRENCY = '$'
    # SALARYの分母(million)
    BASE_SALARY = 1000000

    # ピタゴラス勝率のべき乗
    PYTHAGORIAN_POWER = 1.8

    # 戻り値の件数(固定)
    STATS_LIMIT = 4

    def __init__(self, encode):
        locale.setlocale(locale.LC_NUMERIC, encode)

    def get_base_context(self):
        """
        戻り値定数セット
        :return: 戻り値(固定)
        """
        return {
            'APPLICATION_NAME': APPLICATION_NAME,
            'QUERY_KEY': Stats.QUERY_KEY,
            'SUB_DOMAIN': Stats.SUB_DOMAIN,
            'PLAYER_BATTER': POSITION_BATTER,
            'PLAYER_PITCHER': POSITION_PITCHER,
            'MENU_ENABLE': True,
        }

    def get_pythagoras_dataset(self, teams):
        """
        ピタゴラス勝率を含んだデータセットを返す
        :param teams: Team Model List
        :return: pythagoras % list
        """
        datasets = []
        for t in teams:
            d = {
                'team': t.teamid,
                'name': t.name,
                'div': t.divid,
                'rank': t.rank,
                'g': t.g,
                'w': t.w,
                'l': t.l,
                'r': t.r,
                'er': t.er,
                'win_p': Stats.calc_winning_percentage(t.w, t.g),
                'pyt_p': Stats.calc_pythagorean_expectation(t.r, t.ra)
            }
            d['pyt_p_w'] = round(t.g * d['pyt_p'], 0)
            d['pyt_p_l'] = t.g - d['pyt_p_w']
            datasets.append(d)
        return datasets

    def get_sabr_value_batter(self, player, stats, salary):
        """
        SABR Stats for batter
        :param player: Player master model
        :param stats: Player stats model
        :param salary: Player salary model
        :return: (dict)sabar stats data
        """
        # SABR Data(batter)
        _datasets = {
            'rc_list': [],
            'babip_list': [],
            'dunn_list': []
        }

        for batting in stats:
            year = batting.yearid
            pa = BaseballStats.pa(batting.ab, batting.bb, batting.hbp, batting.sf, batting.sh)
            single = BaseballStats.single(batting.h, batting.hr, batting.number_2b, batting.number_3b)
            rc = BaseballStats.rc(
                batting.h,
                batting.bb,
                batting.hbp,
                batting.cs,
                batting.gidp,
                batting.sf,
                batting.sh,
                batting.sb,
                batting.so,
                batting.ab,
                batting.ibb,
                single,
                batting.number_2b,
                batting.number_3b,
                batting.hr
            )

            _datasets['rc_list'].append(
                {
                    'year': year,
                    'rc': BaseballStats.rc27(rc, batting.ab, batting.h, batting.sh, batting.sf, batting.cs, batting.gidp)
                }
            )

            _datasets['dunn_list'].append(
                {
                    'year': year,
                    'dunn': BaseballStats.adam_dunn_batter(batting.hr, batting.bb, batting.so, pa)
                }
            )
            _datasets['babip_list'].append(
                {
                    'year': year,
                    'avg': BaseballStats.avg(batting.h, batting.ab),
                    'babip': BaseballStats.babip(batting.h, batting.hr, batting.ab, batting.so, batting.sf)
                }
            )
        return _datasets

    def get_sabr_value_pitcher(self, player, stats, salary):
        """
        SABR Stats for batter
        :param player: Player master model
        :param stats: Player stats model
        :param salary: Player salary model
        :return: (dict)sabar stats data
        """
        # SABR Data(pitcher)
        _datasets = {
            'whip_list': [],
            'hr9_list': [],
            'dunn_list': []
        }

        for pitching in stats:
            year = pitching.yearid

            ip = BaseballStats.ip(pitching.ipouts)

            _datasets['whip_list'].append(
                {
                    'year': year,
                    'whip': BaseballStats.whip(pitching.bb, pitching.h, ip)
                }
            )
            _datasets['hr9_list'].append(
                {
                    'year': year,
                    'hr9': BaseballStats.hr9(pitching.hr, ip),
                    'bb9': BaseballStats.bb9(pitching.bb, ip),
                    'so9': BaseballStats.so9(pitching.so, ip)
                }
            )
            _datasets['dunn_list'].append(
                {
                    'year': year,
                    'dunn': BaseballStats.adam_dunn_pitcher(pitching.hr, pitching.bb, pitching.hbp, pitching.so, pitching.bfp)
                }
            )
        return _datasets

    def get_home_value_pitcher(self, player, stats, salary):
        """
        Pitcher profile
        :param player: Player master model
        :param stats: Player stats model
        :param salary: Player salary model
        :return: pitcher profile(dict)
        """
        # statsから必要な値を修得
        w, l, so, er, ipo, year, teams = 0, 0, 0, 0, 0, 0, []
        if len(stats) > 0:
            pitching = stats[0]
            year = pitching.yearid
            w = pitching.w
            l = pitching.l
            so = pitching.so
            er = pitching.er
            ipo = pitching.ipouts
            teams.append(MlbDB.get_last_team(MlbDB.get_pitching(), year, pitching.playerid))

        # ipを計算
        ip = BaseballStats.ip(ipo)

        _prof = self._get_base_profile(player, year, teams, salary)
        _prof['position'] = POSITION_PITCHER.upper()
        _prof['win'] = w
        _prof['lose'] = l
        _prof['era'] = BaseballStats.era(er, ip)
        _prof['so'] = so
        return _prof

    def get_home_value_batter(self, player, stats, salary):
        """
        Batter profile
        :param player: Player master model
        :param stats: Player stats model
        :param salary: Player salary model
        :return: batter profile(dict)
        """
        # statsから必要な値を修得
        hr, rbi, h, ab, year, teams = 0, 0, 0, 0, 0, []
        if len(stats) > 0:
            batter = stats[0]
            year = batter.yearid
            h = batter.h
            ab = batter.ab
            hr = batter.hr
            rbi = rbi + batter.rbi
            teams.append(MlbDB.get_last_team(MlbDB.get_batting(), year, batter.playerid))

        _prof = self._get_base_profile(player, year, teams, salary)
        _prof['position'] = POSITION_BATTER.upper()
        _prof['avg'] = BaseballStats.avg(h, ab)
        _prof['hr'] = hr
        _prof['rbi'] = rbi
        return _prof

    def _get_base_profile(self, player, year, teams, salary):
        """
        Profile(Base)
        :param player: Player master model
        :param year: Last play year
        :param teams: teams
        :param salary: Player salary model
        :return: profile(dict)
        """
        return {
            'year': year,
            'team': ','.join(teams),
            'age': Stats.calc_age(player.birthyear),
            'birthday': '%d/%d/%d' % (player.birthyear, player.birthmonth, player.birthday),
            'salary': Stats._calc_salary(salary),
            'country': player.birthcountry,
            'city': player.birthcity,
            'bats': player.bats,
            'throws': player.throws,
            'display_name': ' '.join([player.namefirst, player.namelast])

        }

    @classmethod
    def _calc_salary(cls, salary):
        """
        年俸計算
        :param salary: salary model list
        :return:
        """
        sal = 0
        if len(salary) > 0:
            sal = salary[0].salary
        return Stats._get_salary(sal)

    @classmethod
    def _get_salary(cls, money):
        """
        表示用のsalary
        :param money: salary
        :return: (str) salary
        """
        # SALARY編集
        if money == None or money == '':
            return '?'
        _money = int(money)
        if _money >= Stats.BASE_SALARY:
            return '%s%sM' % (Stats.CURRENCY, "{:n}".format(_money / Stats.BASE_SALARY))
        else:
            return '%s%s' % (Stats.CURRENCY, "{:n}".format(_money))

    @classmethod
    def calc_age(cls, year):
        """
        calculate age
        :param year: (int)birth year
        :return: (int) age
        """
        now = datetime.now()
        return now.year - year

    @classmethod
    def calc_winning_percentage(cls, w, g):
        """
        勝率
        :param w: win
        :param g: games
        :return: winning percentage
        """
        return round(w / g, 3)

    @classmethod
    def calc_pythagorean_expectation(cls, r, ra):
        """
        ピタゴラス勝率
        :param r: Runs Scored
        :param ra: Runs Allowed
        :return: pythagorean expectation
        """
        r_power = r ** Stats.PYTHAGORIAN_POWER
        ra_power = ra ** Stats.PYTHAGORIAN_POWER
        return round(r_power / (r_power + ra_power), 3)

    @classmethod
    def exists_player(cls, firstname, lastname):
        """
        選手がいるかいないか
        :param firstname: Player's First name
        :param lastname: Player's Last name
        :return: True(exists) or False(not exists)
        """
        if MlbDB.player().filter(namefirst=firstname).filter(namelast=lastname).count() == 1:
            return True
        else:
            return False<|endoftext|>


<commit_before>import logging

from django.contrib import auth
from django.views.decorators.csrf import csrf_exempt
from djangosaml2.conf import get_config
from djangosaml2.signals import post_authenticated
from djangosaml2.utils import get_custom_setting
from rest_framework import status
from rest_framework.authtoken.models import Token
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.views import APIView
from saml2.client import Saml2Client

from nodeconductor import __version__
from nodeconductor.core.log import EventLoggerAdapter
from nodeconductor.core.serializers import AuthTokenSerializer, Saml2ResponseSerializer

logger = logging.getLogger(__name__)
event_logger = EventLoggerAdapter(logger)

class ObtainAuthToken(APIView):
    """
    Api view loosely based on DRF's default ObtainAuthToken,
    but with the responses formats and status codes aligned with BasicAuthentication behavior.
    """
    throttle_classes = ()
    permission_classes = ()
    queryset = Token.objects.all()
    serializer_class = AuthTokenSerializer

    def post(self, request):
        serializer = self.serializer_class(data=request.DATA)
        if serializer.is_valid():
            user = serializer.object['user']
            token, created = Token.objects.get_or_create(user=user)
            event_logger.info(
                "User '%s' with full name '%s' authenticated successfully with username and password.",
                user.username, user.full_name,
                extra={'user': user, 'event_type': 'auth_logged_in_with_username'})
            logger.debug('Returning token for successful login of user %s', user)
            return Response({'token': token.key})

        errors = dict(serializer.errors)

        try:
            non_field_errors = errors.pop('non_field_errors')
            errors['detail'] = non_field_errors[0]
        except (KeyError, IndexError):
            pass

        logger.debug('Failed session token retrieval due to %s', errors)
        return Response(errors, status=status.HTTP_401_UNAUTHORIZED)

obtain_auth_token = ObtainAuthToken.as_view()

class Saml2AuthView(APIView):
    throttle_classes = ()
    permission_classes = ()
    serializer_class = Saml2ResponseSerializer

    @csrf_exempt
    def post(self, request):
        """SAML Authorization Response endpoint

        The IdP will send its response to this view, which
        will process it with pysaml2 help and log the user
        in using the custom Authorization backend
        djangosaml2.backends.Saml2Backend that should be
        enabled in the settings.py
        """
        serializer = self.serializer_class(data=request.DATA)
        if not serializer.is_valid():
            errors = dict(serializer.errors)

            try:
                non_field_errors = errors.pop('non_field_errors')
                errors['detail'] = non_field_errors[0]
            except (KeyError, IndexError):
                pass

            return Response(errors, status=status.HTTP_401_UNAUTHORIZED)

        attribute_mapping = get_custom_setting(
            'SAML_ATTRIBUTE_MAPPING', {'uid': ('username', )})
        create_unknown_user = get_custom_setting(
            'SAML_CREATE_UNKNOWN_USER', True)

        conf = get_config(request=request)
        client = Saml2Client(conf, logger=logger)

        post = {'SAMLResponse': serializer.object['saml2response']}

        # process the authentication response
        # noinspection PyBroadException
        try:
            response = client.response(
                post,
                outstanding=None,  # Rely on allow_unsolicited setting
                decode=False,      # The response is already base64 decoded
            )
        except Exception as e:
            logger.error('SAML response parsing failed %s' % e)
            response = None

        if response is None:
            return Response(
                {'saml2response': 'SAML2 response has errors.'},
                status=status.HTTP_401_UNAUTHORIZED
            )

        # authenticate the remote user
        session_info = response.session_info()

        user = auth.authenticate(
            session_info=session_info,
            attribute_mapping=attribute_mapping,
            create_unknown_user=create_unknown_user,
        )
        if user is None:
            logger.info('Authentication with SAML token has failed, user not found')
            return Response(
                {'detail': 'SAML2 authentication failed'},
                status=status.HTTP_401_UNAUTHORIZED
            )

        post_authenticated.send_robust(sender=user, session_info=session_info)

        token, _ = Token.objects.get_or_create(user=user)
        event_logger.info(
            "User '%s' with full name '%s' authenticated successfully with Omani PKI.",
            user.username, user.full_name,
            extra={'user': user, 'event_type': 'auth_logged_in_with_pki'})
        logger.info('Authenticated with SAML token. Returning token for successful login of user %s', user)
        return Response({'token': token.key})

assertion_consumer_service = Saml2AuthView.as_view()

@api_view(['GET'])
def version_detail(request):
    """Retrieve version of the application"""

    return Response({'version': __version__})
<commit_msg>Remove quotes from the log messages<commit_after>import logging

from django.contrib import auth
from django.views.decorators.csrf import csrf_exempt
from djangosaml2.conf import get_config
from djangosaml2.signals import post_authenticated
from djangosaml2.utils import get_custom_setting
from rest_framework import status
from rest_framework.authtoken.models import Token
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.views import APIView
from saml2.client import Saml2Client

from nodeconductor import __version__
from nodeconductor.core.log import EventLoggerAdapter
from nodeconductor.core.serializers import AuthTokenSerializer, Saml2ResponseSerializer

logger = logging.getLogger(__name__)
event_logger = EventLoggerAdapter(logger)

class ObtainAuthToken(APIView):
    """
    Api view loosely based on DRF's default ObtainAuthToken,
    but with the responses formats and status codes aligned with BasicAuthentication behavior.
    """
    throttle_classes = ()
    permission_classes = ()
    queryset = Token.objects.all()
    serializer_class = AuthTokenSerializer

    def post(self, request):
        serializer = self.serializer_class(data=request.DATA)
        if serializer.is_valid():
            user = serializer.object['user']
            token, created = Token.objects.get_or_create(user=user)
            event_logger.info(
                "User %s with full name %s authenticated successfully with username and password.",
                user.username, user.full_name,
                extra={'user': user, 'event_type': 'auth_logged_in_with_username'})
            logger.debug('Returning token for successful login of user %s', user)
            return Response({'token': token.key})

        errors = dict(serializer.errors)

        try:
            non_field_errors = errors.pop('non_field_errors')
            errors['detail'] = non_field_errors[0]
        except (KeyError, IndexError):
            pass

        logger.debug('Failed session token retrieval due to %s', errors)
        return Response(errors, status=status.HTTP_401_UNAUTHORIZED)

obtain_auth_token = ObtainAuthToken.as_view()

class Saml2AuthView(APIView):
    throttle_classes = ()
    permission_classes = ()
    serializer_class = Saml2ResponseSerializer

    @csrf_exempt
    def post(self, request):
        """SAML Authorization Response endpoint

        The IdP will send its response to this view, which
        will process it with pysaml2 help and log the user
        in using the custom Authorization backend
        djangosaml2.backends.Saml2Backend that should be
        enabled in the settings.py
        """
        serializer = self.serializer_class(data=request.DATA)
        if not serializer.is_valid():
            errors = dict(serializer.errors)

            try:
                non_field_errors = errors.pop('non_field_errors')
                errors['detail'] = non_field_errors[0]
            except (KeyError, IndexError):
                pass

            return Response(errors, status=status.HTTP_401_UNAUTHORIZED)

        attribute_mapping = get_custom_setting(
            'SAML_ATTRIBUTE_MAPPING', {'uid': ('username', )})
        create_unknown_user = get_custom_setting(
            'SAML_CREATE_UNKNOWN_USER', True)

        conf = get_config(request=request)
        client = Saml2Client(conf, logger=logger)

        post = {'SAMLResponse': serializer.object['saml2response']}

        # process the authentication response
        # noinspection PyBroadException
        try:
            response = client.response(
                post,
                outstanding=None,  # Rely on allow_unsolicited setting
                decode=False,      # The response is already base64 decoded
            )
        except Exception as e:
            logger.error('SAML response parsing failed %s' % e)
            response = None

        if response is None:
            return Response(
                {'saml2response': 'SAML2 response has errors.'},
                status=status.HTTP_401_UNAUTHORIZED
            )

        # authenticate the remote user
        session_info = response.session_info()

        user = auth.authenticate(
            session_info=session_info,
            attribute_mapping=attribute_mapping,
            create_unknown_user=create_unknown_user,
        )
        if user is None:
            logger.info('Authentication with SAML token has failed, user not found')
            return Response(
                {'detail': 'SAML2 authentication failed'},
                status=status.HTTP_401_UNAUTHORIZED
            )

        post_authenticated.send_robust(sender=user, session_info=session_info)

        token, _ = Token.objects.get_or_create(user=user)
        event_logger.info(
            "User %s with full name %s authenticated successfully with Omani PKI.",
            user.username, user.full_name,
            extra={'user': user, 'event_type': 'auth_logged_in_with_pki'})
        logger.info('Authenticated with SAML token. Returning token for successful login of user %s', user)
        return Response({'token': token.key})

assertion_consumer_service = Saml2AuthView.as_view()

@api_view(['GET'])
def version_detail(request):
    """Retrieve version of the application"""

    return Response({'version': __version__})
<|endoftext|>


<commit_before>import json
from subprocess import call
from pprint import pprint

from django.db import transaction
from django.test import TransactionTestCase
from django.contrib.auth.models import User
from tastypie.test import ResourceTestCase
from django.test import Client
# from django.conf.test_settings import PROJECT_ROOT
from pandas import read_csv, notnull

from datapoints.models import *
from source_data.models import *
from datapoints.cache_tasks import CacheRefresh

class CacheRefreshTestCase(TransactionTestCase):
# class CacheRefreshTestCase(TestCase):

    '''
        from datapoints.cache_tasks import CacheRefresh
        mr = CacheRefresh()

        ## or ##

        from datapoints.cache_tasks import CacheRefresh
        from datapoints.models import DataPoint, Region
        r_ids = Region.objects.filter(parent_region_id = 12907).values_list('id',flat=True)
        dp_ids = DataPoint.objects.filter(region_id__in=r_ids,campaign_id=111,indicator_id__in=[55]).values_list('id',flat=True)
        mr = CacheRefresh(list(dp_ids))
    '''

    def __init__(self, *args, **kwargs):

        super(CacheRefreshTestCase, self).__init__(*args, **kwargs)

    def set_up(self):

        data_df = read_csv('datapoints/tests/_data/calc_data.csv')

        self.changed_by_id = 1

        self.test_df = data_df[data_df['is_raw'] == 1]
        self.target_df = data_df[data_df['is_raw'] == 0]

        self.build_db() # builds sprocs and views needed to test cache refresh
        self.create_metadata()

    def build_db(self):
        '''
        please fix me
        '''

        ## remove the build_test_db script and pass a $DB param to build_db.sh
        call(["bash" ,"/Users/johndingee_seed/code/UF04/polio/bin/build_test_db.sh"])

    def create_metadata(self):
        '''
        Creating the Indicator, Region, Campaign, meta data needed for the
        system to aggregate / caclulate.
        '''
        campaign_df = read_csv('datapoints/tests/_data/campaigns.csv')
        region_df= read_csv('datapoints/tests/_data/regions.csv')
        indicator_df = read_csv('datapoints/tests/_data/indicators.csv')
        calc_indicator_df = read_csv\
            ('datapoints/tests/_data/calculated_indicator_component.csv')

        user_id = User.objects.create_user('test','john@john.com', 'test').id

        office_id = Office.objects.create(id=1,name='test').id

        source_id1 = Source.objects.create(id=1,source_name='test1',source_description='test1').id
        source_id2 = Source.objects.create(id=2,source_name='test2',source_description='test2').id

        cache_job_id = CacheJob.objects.create(id = -2,date_attempted = '2015-01-01', is_error = False)

        status_id = ProcessStatus.objects.create(
                status_text = 'test',
                status_description = 'test').id

        document_id = Document.objects.create(
            doc_text = 'test',
            created_by_id = user_id,
            source_id = source_id1,
            guid = 'test').id

        sdp_id = SourceDataPoint.objects.create(
            id = -1,
            document_id = document_id,
            row_number = 0,
            source_id = source_id1,
            status_id = status_id).id

        region_type1 = RegionType.objects.create(id=1,name="country")
        region_type2 = RegionType.objects.create(id=2,name="settlement")
        region_type3 = RegionType.objects.create(id=3,name="province")
        region_type4 = RegionType.objects.create(id=4,name="district")
        region_type5 = RegionType.objects.create(id=5,name="sub-district")

        campaign_type = CampaignType.objects.create(id=1,name="test")

        region_ids = self.model_df_to_data(region_df,Region)
        campaign_ids = self.model_df_to_data(campaign_df,Campaign)
        indicator_ids = self.model_df_to_data(indicator_df,Indicator)
        calc_indicator_ids = self.model_df_to_data(calc_indicator_df,\
            CalculatedIndicatorComponent)

    def model_df_to_data(self,model_df,model):

        meta_ids = []

        non_null_df = model_df.where((notnull(model_df)), None)
        list_of_dicts = non_null_df.transpose().to_dict()

        for row_ix, row_dict in list_of_dicts.iteritems():

            row_id = model.objects.create(**row_dict)
            meta_ids.append(row_id)

        return meta_ids

    def create_raw_datapoints(self):

        for row_ix,row_data in self.test_df.iterrows():

            dp_id = self.create_datapoint(row_data.region_id, row_data\
                .campaign_id, row_data.indicator_id, row_data.value)

    def create_datapoint(self, region_id, campaign_id, indicator_id, value):
        '''
        Right now this is being performed as a database insert.  I would like to
        Test this against the data entry resource, but this will do for now
        in order to test caching.
        '''

        dp_id = DataPoint.objects.create(
            region_id = region_id,
            campaign_id = campaign_id,
            indicator_id = indicator_id,
            value = value,
            changed_by_id = self.changed_by_id,
            source_datapoint_id = -1
        ).id

        return dp_id

    # def test_basic(self):
    #
    #     self.set_up()
    #     self.create_raw_datapoints()
    #
    #     cr = CacheRefresh()
    #
    #     for ix, row in self.target_df.iterrows():
    #
    #         region_id, campaign_id, indicator_id = int(row.region_id),\
    #            int(row.campaign_id),int(row.indicator_id)

    #         actual_value = self.get_dwc_value(region_id, campaign_id,\
    #             indicator_id)
    #
    #         self.assertEqual(row.value,actual_value)

    def test_agg(self):

        raw_indicator_id = 22
        campaign_id = 111
        raw_region_id = 12939
        agg_region_id = 12907

        new_dp_val = 1.02

        self.set_up()
        self.create_raw_datapoints()

        agg_value_target = self.test_df = self.test_df[self.test_df['indicator_id'] ==\
             raw_indicator_id]['value'].sum()

        #################################################
        ## only refresh the cache for one datapoint_id ##
        ## and make sure agg uses all child data below ##
        #################################################

        print 'CACHE_REFRESH #2\n' * 5

        with transaction.atomic():

            # update the db so we only process one record #
            dp_curs = DataPoint.objects.raw('''

                --UPDATE datapoint
                --SET cache_job_id = -2
                    --,value = %s
                --WHERE region_id != %s
                --AND indicator_id != %s
                --AND campaign_id != %s;

                SELECT id from datapoint limit 1;

            ''',[new_dp_val,raw_indicator_id,campaign_id,raw_region_id])

            dp_val = [dp.id for dp in dp_curs]

        cr = CacheRefresh()

        actual_value = self.get_dwc_value(agg_region_id,campaign_id,\
            raw_indicator_id)

        ## test that the change effected the parent (POLIO-491)
        self.assertEqual(actual_value,agg_value_target)

    def get_dwc_value(self,region_id,campaign_id,indicator_id):
        '''
        This testings the API for a row in the target dataframe and returns
        the corresponding value

        Testing the Value agains the API test_client currenlty is not working.
        Instead I will query the datapoint_with_computed table and ensure
        later that the dataopint_abstracted transformation is working properly
        '''

        actual_value = DataPointComputed.objects.get(
            region_id = region_id,
            indicator_id = indicator_id,
            campaign_id = campaign_id
        ).value

        # target_url = \
        # '/api/v1/datapoint/?region__in=%s&campaign__in=%s&indicator__in=%s'\
        # % (int(row.region_id),int(row.campaign_id),int(row.indicator_id))

        # c = Client()
        # resp = c.get(target_url,format='json',follow=True)
        # response_data = json.loads(resp.content)['objects']

        # print response_data

        # actual_value = float(response_data[0]['value'])

        return actual_value
<commit_msg>set up agg test case for polio-491<commit_after>import json
from subprocess import call
from pprint import pprint

from django.db import transaction
from django.test import TransactionTestCase
from django.contrib.auth.models import User
from tastypie.test import ResourceTestCase
from django.test import Client
# from django.conf.test_settings import PROJECT_ROOT
from pandas import read_csv, notnull

from datapoints.models import *
from source_data.models import *
from datapoints.cache_tasks import CacheRefresh

class CacheRefreshTestCase(TransactionTestCase):
# class CacheRefreshTestCase(TestCase):

    '''
        from datapoints.cache_tasks import CacheRefresh
        mr = CacheRefresh()

        ## or ##

        from datapoints.cache_tasks import CacheRefresh
        from datapoints.models import DataPoint, Region
        r_ids = Region.objects.filter(parent_region_id = 12907).values_list('id',flat=True)
        dp_ids = DataPoint.objects.filter(region_id__in=r_ids,campaign_id=111,indicator_id__in=[55]).values_list('id',flat=True)
        mr = CacheRefresh(list(dp_ids))
    '''

    def __init__(self, *args, **kwargs):

        super(CacheRefreshTestCase, self).__init__(*args, **kwargs)

    def set_up(self):

        data_df = read_csv('datapoints/tests/_data/calc_data.csv')

        self.changed_by_id = 1

        self.test_df = data_df[data_df['is_raw'] == 1]
        self.target_df = data_df[data_df['is_raw'] == 0]

        self.build_db() # builds sprocs and views needed to test cache refresh
        self.create_metadata()

    def build_db(self):
        '''
        please fix me
        '''

        ## remove the build_test_db script and pass a $DB param to build_db.sh
        call(["bash" ,"/Users/johndingee_seed/code/UF04/polio/bin/build_test_db.sh"])

    def create_metadata(self):
        '''
        Creating the Indicator, Region, Campaign, meta data needed for the
        system to aggregate / caclulate.
        '''
        campaign_df = read_csv('datapoints/tests/_data/campaigns.csv')
        region_df= read_csv('datapoints/tests/_data/regions.csv')
        indicator_df = read_csv('datapoints/tests/_data/indicators.csv')
        calc_indicator_df = read_csv\
            ('datapoints/tests/_data/calculated_indicator_component.csv')

        user_id = User.objects.create_user('test','john@john.com', 'test').id

        office_id = Office.objects.create(id=1,name='test').id

        source_id1 = Source.objects.create(id=1,source_name='test1',source_description='test1').id
        source_id2 = Source.objects.create(id=2,source_name='test2',source_description='test2').id

        cache_job_id = CacheJob.objects.create(id = -2,date_attempted = '2015-01-01', is_error = False)

        status_id = ProcessStatus.objects.create(
                status_text = 'test',
                status_description = 'test').id

        document_id = Document.objects.create(
            doc_text = 'test',
            created_by_id = user_id,
            source_id = source_id1,
            guid = 'test').id

        sdp_id = SourceDataPoint.objects.create(
            id = -1,
            document_id = document_id,
            row_number = 0,
            source_id = source_id1,
            status_id = status_id).id

        region_type1 = RegionType.objects.create(id=1,name="country")
        region_type2 = RegionType.objects.create(id=2,name="settlement")
        region_type3 = RegionType.objects.create(id=3,name="province")
        region_type4 = RegionType.objects.create(id=4,name="district")
        region_type5 = RegionType.objects.create(id=5,name="sub-district")

        campaign_type = CampaignType.objects.create(id=1,name="test")

        region_ids = self.model_df_to_data(region_df,Region)
        campaign_ids = self.model_df_to_data(campaign_df,Campaign)
        indicator_ids = self.model_df_to_data(indicator_df,Indicator)
        calc_indicator_ids = self.model_df_to_data(calc_indicator_df,\
            CalculatedIndicatorComponent)

    def model_df_to_data(self,model_df,model):

        meta_ids = []

        non_null_df = model_df.where((notnull(model_df)), None)
        list_of_dicts = non_null_df.transpose().to_dict()

        for row_ix, row_dict in list_of_dicts.iteritems():

            row_id = model.objects.create(**row_dict)
            meta_ids.append(row_id)

        return meta_ids

    def create_raw_datapoints(self):

        for row_ix,row_data in self.test_df.iterrows():

            dp_id = self.create_datapoint(row_data.region_id, row_data\
                .campaign_id, row_data.indicator_id, row_data.value)

    def create_datapoint(self, region_id, campaign_id, indicator_id, value):
        '''
        Right now this is being performed as a database insert.  I would like to
        Test this against the data entry resource, but this will do for now
        in order to test caching.
        '''

        dp_id = DataPoint.objects.create(
            region_id = region_id,
            campaign_id = campaign_id,
            indicator_id = indicator_id,
            value = value,
            changed_by_id = self.changed_by_id,
            source_datapoint_id = -1
        ).id

        return dp_id

    # def test_basic(self):
    #
    #     self.set_up()
    #     self.create_raw_datapoints()
    #
    #     cr = CacheRefresh()
    #
    #     for ix, row in self.target_df.iterrows():
    #
    #         region_id, campaign_id, indicator_id = int(row.region_id),\
    #            int(row.campaign_id),int(row.indicator_id)

    #         actual_value = self.get_dwc_value(region_id, campaign_id,\
    #             indicator_id)
    #
    #         self.assertEqual(row.value,actual_value)

    def test_agg(self):
        '''
        When requesting aggregation for one child, ensure that the aggregation
        looks for all children, not just those with cache_job_id = -1 See
        (POLIO-491)
        '''

        raw_indicator_id = 22
        campaign_id = 111
        raw_region_id = 12939
        agg_region_id = 12907
        new_dp_val = 1.02

        self.set_up()
        self.create_raw_datapoints()

        agg_value_target = self.test_df = self.test_df[self.test_df['indicator_id'] ==\
             raw_indicator_id]['value'].sum()

        #################################################
        ## only refresh the cache for one datapoint_id ##
        ## and make sure agg uses all child data below ##
        #################################################

        dp_id_to_refresh = DataPoint.objects.filter(
            region_id = raw_region_id,
            campaign_id = campaign_id ,
            indicator_id = raw_indicator_id
        ).values_list('id',flat=True)

        cr = CacheRefresh(datapoint_id_list=list(dp_id_to_refresh))

        actual_value = self.get_dwc_value(agg_region_id,campaign_id,\
            raw_indicator_id)

        self.assertEqual(actual_value,agg_value_target)

    def get_dwc_value(self,region_id,campaign_id,indicator_id):
        '''
        This testings the API for a row in the target dataframe and returns
        the corresponding value

        Testing the Value agains the API test_client currenlty is not working.
        Instead I will query the datapoint_with_computed table and ensure
        later that the dataopint_abstracted transformation is working properly
        '''

        actual_value = DataPointComputed.objects.get(
            region_id = region_id,
            indicator_id = indicator_id,
            campaign_id = campaign_id
        ).value

        # target_url = \
        # '/api/v1/datapoint/?region__in=%s&campaign__in=%s&indicator__in=%s'\
        # % (int(row.region_id),int(row.campaign_id),int(row.indicator_id))

        # c = Client()
        # resp = c.get(target_url,format='json',follow=True)
        # response_data = json.loads(resp.content)['objects']

        # print response_data

        # actual_value = float(response_data[0]['value'])

        return actual_value
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Anomaly-related algorithms."""

import numpy

from nupic.algorithms.anomaly_likelihood import AnomalyLikelihood

def pureAnomaly(activeColumns, prevPredictedColumns):
    """the pure anomaly score

    computed as diff of current active columns and columns predicted from previous round
    @param activeColumns: array of active column indices
    @param prevPredictedColumns: array of columns indices predicted in previous step
    @return anomaly score 0..1 (float)
    """
    nActiveColumns = len(activeColumns)
    if nActiveColumns > 0:
      # Test whether each element of a 1-D array is also present in a second
      # array. Sum to get the total # of columns that are active and were
      # predicted.
      score = numpy.sum(numpy.in1d(activeColumns, prevPredictedColumns))
      # Get the percent of active columns that were NOT predicted, that is
      # our anomaly score.
      score = (nActiveColumns - score) / float(nActiveColumns)
    elif len(prevPredictedColumns) > 0:
      # There were predicted columns but none active.
      score = 1.0
    else:
      # There were no predicted or active columns.
      score = 0.0
    return score

class Anomaly(object):
  """basic class that computes anomaly"""

  # anomaly modes supported
  MODE_PURE = "pure"
  MODE_LIKELIHOOD = "likelihood"
  MODE_WEIGHTED = "weighted"

  def __init__(self, useTP = None, slidingWindowSize = None, anomalyMode="pure"):
    """
    @param (optional) useTP -- tp temporal pooler instance used
    @param (optional) slidingWindowSize -- enables moving average on final
            anomaly score; how many elements are summed up, sliding window size; int >= 0
    @param (optional) anomalyMode -- (string) which way to use to compute anomaly;
                      possible values are:
                           -- "pure" -- the default, how much anomal the value is;
                                        float 0..1 where 1=totally unexpected
                           -- "likelihood" -- uses the anomaly_likelihood code;
                                        models probability of receiving this value and anomalyScore;
                                        used in Grok
                           -- "weighted" -- "pure" anomaly weighted by "likelihood" (anomaly * likelihood)
    """
#    self.pureAnomaly = staticmethod(pureAnomaly)

    # using TP
    self._tp = useTP
    if self._tp is not None:
      self._prevPredictedColumns = numpy.array([])
    # using cumulative anomaly , sliding window
    self._windowSize = slidingWindowSize
    if self._windowSize is not None:
      self._buf = numpy.array([0] * self._windowSize, dtype=numpy.float) #sliding window buffer
      self._i = 0 # index pointer to actual position
    # mode
    self._mode = anomalyMode
    if self._mode == Anomaly.MODE_LIKELIHOOD:
      self._likelihood = AnomalyLikelihood() # probabilistic anomaly

  def computeAnomalyScore(self, activeColumns, prevPredictedColumns, value=None, timestamp=None):
    """Compute the anomaly score as the percent of active columns not predicted.

    @param activeColumns: array of active column indices
    @param prevPredictedColumns: array of columns indices predicted in previous step (ignored with useTP != None)
    @param value: (optional) input value, that is what activeColumns represent; used in anomaly-likelihood
    @param timestamp: (optional) date timestamp when the sample occured; used in anomaly-likelihood
    @return the computed anomaly score; float 0..1
    """

    # using TP provided during init, _prevPredColumns stored internally here
    if self._tp is not None:
      prevPredictedColumns = self._prevPredictedColumns # override the values passed by parameter with the stored value
      self._prevPredictedColumns = self._tp._getTPRegion().getOutputData("topDownOut").nonzero()[0]

    # 1. here is the 'classic' anomaly score
    anomalyScore = pureAnomaly(activeColumns, prevPredictedColumns)

    # compute final anomaly based on selected mode
    if self._mode == Anomaly.MODE_PURE:
      score = anomalyScore
    elif self._mode == Anomaly.MODE_LIKELIHOOD:
      probability = self._likelihood.anomalyProbability(value, anomalyScore, timestamp)
      score = probability
    elif self._mode == Anomaly.MODE_WEIGHTED:
      probability = self._likelihood.anomalyProbability(value, anomalyScore, timestamp)
      score = anomalyScore * probability
    else:
      raise ValueError("Invalid anomaly mode; only supported modes are: \"pure\",
                       \"likelihood\", \"weighted\"; you used:"+self._mode)

    # last, do moving-average if windowSize is set
    if self._windowSize is not None:
      score = self._movingAverage(score)

    return score

  def _movingAverage(self, newElement=None):
    """moving average

    @param newValue (optional) add a new element before computing the avg
    @return moving average of self._windowSize last elements
    """
    if self._windowSize is None:
      raise RuntimeError("Moving average has not been enabled during __init__ (self._windowSize not set)")
    if newElement is not None:
      self._buf[self._i]= newElement
      self._i = (self._i + 1) % self._windowSize
    return self._buf.sum()/float(self._windowSize) # normalize to 0..1

<commit_msg>Anomaly: add checks to init()<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Anomaly-related algorithms."""

import numpy

from nupic.algorithms.anomaly_likelihood import AnomalyLikelihood
from nupic.research.TP import TP

def pureAnomaly(activeColumns, prevPredictedColumns):
    """the pure anomaly score

    computed as diff of current active columns and columns predicted from previous round
    @param activeColumns: array of active column indices
    @param prevPredictedColumns: array of columns indices predicted in previous step
    @return anomaly score 0..1 (float)
    """
    nActiveColumns = len(activeColumns)
    if nActiveColumns > 0:
      # Test whether each element of a 1-D array is also present in a second
      # array. Sum to get the total # of columns that are active and were
      # predicted.
      score = numpy.sum(numpy.in1d(activeColumns, prevPredictedColumns))
      # Get the percent of active columns that were NOT predicted, that is
      # our anomaly score.
      score = (nActiveColumns - score) / float(nActiveColumns)
    elif len(prevPredictedColumns) > 0:
      # There were predicted columns but none active.
      score = 1.0
    else:
      # There were no predicted or active columns.
      score = 0.0
    return score

class Anomaly(object):
  """basic class that computes anomaly"""

  # anomaly modes supported
  MODE_PURE = "pure"
  MODE_LIKELIHOOD = "likelihood"
  MODE_WEIGHTED = "weighted"
  _supportedModes = [MODE_PURE, MODE_LIKELIHOOD, MODE_WEIGHTED]

  def __init__(self, useTP = None, slidingWindowSize = None, anomalyMode="pure"):
    """
    @param (optional) useTP -- tp temporal pooler instance used
    @param (optional) slidingWindowSize -- enables moving average on final
            anomaly score; how many elements are summed up, sliding window size; int >= 0
    @param (optional) anomalyMode -- (string) which way to use to compute anomaly;
                      possible values are:
                           -- "pure" -- the default, how much anomal the value is;
                                        float 0..1 where 1=totally unexpected
                           -- "likelihood" -- uses the anomaly_likelihood code;
                                        models probability of receiving this value and anomalyScore;
                                        used in Grok
                           -- "weighted" -- "pure" anomaly weighted by "likelihood" (anomaly * likelihood)
    """
#    self.pureAnomaly = staticmethod(pureAnomaly)

    # using TP
    self._tp = useTP
    if self._tp is not None and isinstance(self._tp, TP):
      self._prevPredictedColumns = numpy.array([])
    else:
      raise Exception("Anomaly: you've provided instance of TP, but it does not look as a correct temporal pooler object: "+str(type(useTP)))

    # using cumulative anomaly , sliding window
    self._windowSize = slidingWindowSize
    if self._windowSize is not None and self._windowSize > 0:
      self._buf = numpy.array([0] * self._windowSize, dtype=numpy.float) #sliding window buffer
      self._i = 0 # index pointer to actual position
    else:
      raise Exception("Anomaly: if you define slidingWindowSize, it has to be an integer > 0; slidingWindowSize="+str(slidingWindowSize))

    # mode
    self._mode = anomalyMode
    if self._mode == Anomaly.MODE_LIKELIHOOD:
      self._likelihood = AnomalyLikelihood() # probabilistic anomaly
    if not (self._mode in _supportedModes):
      raise ValueError("Invalid anomaly mode; only supported modes are: \"pure\",
                       \"likelihood\", \"weighted\"; you used:"+self._mode)

  def computeAnomalyScore(self, activeColumns, prevPredictedColumns, value=None, timestamp=None):
    """Compute the anomaly score as the percent of active columns not predicted.

    @param activeColumns: array of active column indices
    @param prevPredictedColumns: array of columns indices predicted in previous step (ignored with useTP != None)
    @param value: (optional) input value, that is what activeColumns represent; used in anomaly-likelihood
    @param timestamp: (optional) date timestamp when the sample occured; used in anomaly-likelihood
    @return the computed anomaly score; float 0..1
    """

    # using TP provided during init, _prevPredColumns stored internally here
    if self._tp is not None:
      prevPredictedColumns = self._prevPredictedColumns # override the values passed by parameter with the stored value
      self._prevPredictedColumns = self._tp._getTPRegion().getOutputData("topDownOut").nonzero()[0]

    # 1. here is the 'classic' anomaly score
    anomalyScore = pureAnomaly(activeColumns, prevPredictedColumns)

    # compute final anomaly based on selected mode
    if self._mode == Anomaly.MODE_PURE:
      score = anomalyScore
    elif self._mode == Anomaly.MODE_LIKELIHOOD:
      probability = self._likelihood.anomalyProbability(value, anomalyScore, timestamp)
      score = probability
    elif self._mode == Anomaly.MODE_WEIGHTED:
      probability = self._likelihood.anomalyProbability(value, anomalyScore, timestamp)
      score = anomalyScore * probability

    # last, do moving-average if windowSize is set
    if self._windowSize is not None:
      score = self._movingAverage(score)

    return score

  def _movingAverage(self, newElement=None):
    """moving average

    @param newValue (optional) add a new element before computing the avg
    @return moving average of self._windowSize last elements
    """
    if self._windowSize is None:
      raise RuntimeError("Moving average has not been enabled during __init__ (self._windowSize not set)")
    if newElement is not None:
      self._buf[self._i]= newElement
      self._i = (self._i + 1) % self._windowSize
    return self._buf.sum()/float(self._windowSize) # normalize to 0..1

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db.models import FloatField, CharField as ModelCharField
from django.forms import CharField
from unit_field.units import (Unit, UnitValueCreator,
    UNITS_LENGTH,
    UNITS_SQUARE_MEASURE,
    UNITS_SOLID_MEASURE,
    UNITS_MASS,
    UNITS_TIME,
    UNITS_ELECTRIC_CURRENT,
    UNITS_TEMPERATURE,
    UNITS_AMOUNT_OF_SUBSTANCE,
    UNITS_LUMINOUS_INTENSITY,

    UNITS_ACCELERATION,
    UNITS_ANGLE,
    UNITS_CRACKLE,
    UNITS_CURRENT,
    UNITS_DENSITY,
    UNITS_FORCE,
    UNITS_INERTIA_TORQUE,
    UNITS_JERK,
    UNITS_POTENTIAL,
    UNITS_SNAP,
    UNITS_SPEED,
    UNITS_TORSION,
    UNITS_TORQUE,
    UNITS_VELOCITY,

    UNITS_POWER,
    UNITS_THERMAL_RESISTANCE,
    UNITS_MOTOR_CONSTANT,
    UNITS_FORCE_CONSTANT,
    UNITS_ELECTRICAL_TIME_CONSTANT,
    UNITS_POTENTIAL_CONSTANT,
    UNITS_ELECTRICAL_RESISTANCE,
    UNITS_INDUCTANCE)

from unit_field import forms

try:
    from hashlib import md5
except ImportError:
    from md5 import new as md5

def md5_hexdigest(value):
    return md5(value).hexdigest()

def get_factor(units, unit_id):
    """
    Returns the factor of a Unit-Config
    """
    if not units:
        return None

    for unit in units:
        if unit.id == unit_id:
            return unit.factor

class UnitInputField(FloatField):

    auto_convert = True

    def __init__(self, *args, **kwargs):
        auto_convert = kwargs.pop('auto_convert', None)
        if auto_convert is not None:
            self.auto_convert = auto_convert
        super(UnitInputField, self).__init__(*args, **kwargs)

    def get_prep_value(self, value):
        try:
            return float(value)
        except TypeError, ValueError:
            return 0.0

    def formfield(self, **kwargs):
        if self.auto_convert:
            defaults = {'form_class': forms.UnitInputField }
        else:
            defaults = {'form_class': CharField }
        defaults.update(kwargs)
        return super(UnitInputField, self).formfield(**defaults)

class CalculatedFloatField(FloatField):
    """
    This field is required to define the special behaviour of the
    pre-calculated field, used in UnitField
    """
    units = None

    def get_unit_by_id(self, unit_id):
        """
        returns the unit factor of the desired unit, e.g.:
        get_unit_by_id(u'dm²') ---> 0.01
        """
        get_factor(self.units, unit_id)

    def __init__(self, *args, **kwargs):
        kwargs['editable'] = False
        self.units = kwargs.pop('units', None)
        super(CalculatedFloatField, self).__init__(*args, **kwargs)
        self.default = 0.0

    def pre_save(self, model_instance, add):
        input_field_name = self.attname.replace('_value', '_input')
        unit_field_name = self.attname.replace('_value', '_unit')

        input_value = getattr(model_instance, input_field_name)
        unit_id = getattr(model_instance, unit_field_name)

        unit_value = self.get_unit_by_id(unit_id)

        if (input_value is not None) and (unit_value is not None):
            setattr(model_instance, self.attname, input_value * unit_value)
        else:
            setattr(model_instance, self.attname, 0.0)

        return getattr(model_instance, self.attname)

class UnitField(FloatField):
    """
    a compound field contributes three columns to the model instead of the
    standard single column
    """
    choices = None

    units = []

    auto_convert = True

    verbose_name = u'(UnitField)'

    blank = False

    null = False

    default_unit = None

    validators = []

    def update_choices(self):
        """
        generates choices list, if not set
        """
        if self.choices is None:
            self.choices = []
            for unit in self.units:
                self.choices.append((unit.id, unit.abbrev, ))

    def get_base_unit(self):
        """
        returns the unit with factor == 1.0
        """
        if self.units:
            for unit in self.units:
                if unit.factor == 1.0:
                    return unit
        return None

    def get_base_unit_id(self):
        """
        returns the identifier of the base unit (e.g. "m²")
        """
        base_unit = self.get_base_unit()
        if base_unit is None:
            return None

        return base_unit.id

    def __init__(self, *args, **kwargs):
        if 'choices' in kwargs:
            raise TypeError("%s invalid attribute 'choices'" % (
                self.__class__.__name__, ))
        self.auto_convert = kwargs.pop('auto_convert', True)
        self.verbose_name = kwargs.get('verbose_name')
        self.blank = kwargs.get('blank')
        self.null = kwargs.get('null')
        self.validators = kwargs.get('validators', [])
        kwargs['editable'] = False
        kwargs['default'] = 0.

        self.default_unit = kwargs.pop('default_unit',
            self.get_base_unit_id())
        self.update_choices()
        super(UnitField, self).__init__(*args, **kwargs)

    def contribute_to_class(self, cls, name):
        self.name = name

        self.input_field = UnitInputField(default=0,
            blank=self.blank,
            null=self.null,
            auto_convert=self.auto_convert,
            validators=self.validators,
            verbose_name=self.verbose_name)
        cls.add_to_class("%s_input" % (self.name,), self.input_field)

        # self.unit_field = CharField(default=self.default_unit, choices=self.choices)
        self.unit_field = ModelCharField(max_length=10,
            default=self.default_unit,
            choices=self.choices)
        cls.add_to_class("%s_unit" % (self.name,), self.unit_field)

        self.value_field = CalculatedFloatField(default=0.0,
            units=self.units,
            blank=self.blank,
            null=self.null)
        cls.add_to_class("%s_value" % (self.name,), self.value_field)

        self.key = md5_hexdigest(self.name)

        field = UnitValueCreator(self)

        setattr(cls, name, field)

    def get_db_prep_save(self, value):
        pass

    def get_db_prep_lookup(self, lookup_type, value):
        raise NotImplementedError(self.get_db_prep_lookup)

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.UnitField}
        defaults.update(kwargs)
        return super(UnitField, self).formfield(**defaults)

# Fields for Base Units

class LengthField(UnitField):
    units = UNITS_LENGTH

class SquareMeasureField(UnitField):
    units = UNITS_SQUARE_MEASURE

class SolidMeasureField(UnitField):
    units = UNITS_SOLID_MEASURE

class MassField(UnitField):
    units = UNITS_MASS

class TimeField(UnitField):
    units = UNITS_TIME

class TemperatureField(UnitField):
    units = UNITS_TEMPERATURE

class AmountOfSubstanceField(UnitField):
    units = UNITS_AMOUNT_OF_SUBSTANCE

class LuminousIntensityField(UnitField):
    units = UNITS_LUMINOUS_INTENSITY

# Fields for Derived Units

class AccelerationField(UnitField):
    units = UNITS_ACCELERATION

class AngleField(UnitField):
    units = UNITS_ANGLE

class CrackleField(UnitField):
    units = UNITS_CRACKLE

class CurrentField(UnitField):
    units = UNITS_CURRENT

class DensityField(UnitField):
    units = UNITS_DENSITY

class ForceField(UnitField):
    units = UNITS_FORCE

class InertiaTorqueField(UnitField):
    units = UNITS_INERTIA_TORQUE

class JerkField(UnitField):
    units = UNITS_JERK

class PotentialField(UnitField):
    units = UNITS_POTENTIAL

class SnapField(UnitField):
    units = UNITS_SNAP

class SpeedField(UnitField):
    units = UNITS_SPEED

class TorqueField(UnitField):
    units = UNITS_TORQUE

class VelocityField(UnitField):
    units = UNITS_VELOCITY

class TorsionField(UnitField):
    units = UNITS_TORSION

class PowerField(UnitField):
    units = UNITS_POWER

class ThermalResistanceField(UnitField):
    units = UNITS_THERMAL_RESISTANCE

class ElectricalTimeConstantField(UnitField):
    units = UNITS_ELECTRICAL_TIME_CONSTANT

class MotorConstantField(UnitField):
    units = UNITS_MOTOR_CONSTANT

class ForceConstantField(UnitField):
    units = UNITS_FORCE_CONSTANT

class PotentialConstantField(UnitField):
    units = UNITS_POTENTIAL_CONSTANT

class ResistanceField(UnitField):
    units = UNITS_ELECTRICAL_RESISTANCE

class InductanceField(UnitField):
    units = UNITS_INDUCTANCE

try:
    from south.modelsinspector import add_introspection_rules
    rules = [((CalculatedFloatField, ), [], {}),]
    add_introspection_rules(rules, [
        "^unit_field\.fields"])
except ImportError:
    pass
<commit_msg>fixed serious bug (wrong usage of function get_factor(...))<commit_after># -*- coding: utf-8 -*-
from django.db.models import FloatField, CharField as ModelCharField
from django.forms import CharField
from unit_field.units import (Unit, UnitValueCreator,
    UNITS_LENGTH,
    UNITS_SQUARE_MEASURE,
    UNITS_SOLID_MEASURE,
    UNITS_MASS,
    UNITS_TIME,
    UNITS_ELECTRIC_CURRENT,
    UNITS_TEMPERATURE,
    UNITS_AMOUNT_OF_SUBSTANCE,
    UNITS_LUMINOUS_INTENSITY,

    UNITS_ACCELERATION,
    UNITS_ANGLE,
    UNITS_CRACKLE,
    UNITS_CURRENT,
    UNITS_DENSITY,
    UNITS_FORCE,
    UNITS_INERTIA_TORQUE,
    UNITS_JERK,
    UNITS_POTENTIAL,
    UNITS_SNAP,
    UNITS_SPEED,
    UNITS_TORSION,
    UNITS_TORQUE,
    UNITS_VELOCITY,

    UNITS_POWER,
    UNITS_THERMAL_RESISTANCE,
    UNITS_MOTOR_CONSTANT,
    UNITS_FORCE_CONSTANT,
    UNITS_ELECTRICAL_TIME_CONSTANT,
    UNITS_POTENTIAL_CONSTANT,
    UNITS_ELECTRICAL_RESISTANCE,
    UNITS_INDUCTANCE)

from unit_field import forms

try:
    from hashlib import md5
except ImportError:
    from md5 import new as md5

def md5_hexdigest(value):
    return md5(value).hexdigest()

def get_factor(units, unit_id):
    """
    Returns the factor of a Unit-Config
    """
    if not units:
        return None

    for unit in units:
        if unit.id == unit_id:
            return unit.factor

class UnitInputField(FloatField):

    auto_convert = True

    def __init__(self, *args, **kwargs):
        auto_convert = kwargs.pop('auto_convert', None)
        if auto_convert is not None:
            self.auto_convert = auto_convert
        super(UnitInputField, self).__init__(*args, **kwargs)

    def get_prep_value(self, value):
        try:
            return float(value)
        except TypeError, ValueError:
            return 0.0

    def formfield(self, **kwargs):
        if self.auto_convert:
            defaults = {'form_class': forms.UnitInputField }
        else:
            defaults = {'form_class': CharField }
        defaults.update(kwargs)
        return super(UnitInputField, self).formfield(**defaults)

class CalculatedFloatField(FloatField):
    """
    This field is required to define the special behaviour of the
    pre-calculated field, used in UnitField
    """
    units = None

    def get_unit_by_id(self, unit_id):
        """
        returns the unit factor of the desired unit, e.g.:
        get_unit_by_id(u'dm²') ---> 0.01
        """
        return get_factor(self.units, unit_id)

    def __init__(self, *args, **kwargs):
        kwargs['editable'] = False
        self.units = kwargs.pop('units', None)
        super(CalculatedFloatField, self).__init__(*args, **kwargs)
        self.default = 0.0

    def pre_save(self, model_instance, add):
        input_field_name = self.attname.replace('_value', '_input')
        unit_field_name = self.attname.replace('_value', '_unit')

        input_value = getattr(model_instance, input_field_name)
        unit_id = getattr(model_instance, unit_field_name)

        unit_value = self.get_unit_by_id(unit_id)

        if (input_value is not None) and (unit_value is not None):
            setattr(model_instance, self.attname, input_value * unit_value)
        else:
            setattr(model_instance, self.attname, 0.0)

        return getattr(model_instance, self.attname)

class UnitField(FloatField):
    """
    a compound field contributes three columns to the model instead of the
    standard single column
    """
    choices = None

    units = []

    auto_convert = True

    verbose_name = u'(UnitField)'

    blank = False

    null = False

    default_unit = None

    validators = []

    def update_choices(self):
        """
        generates choices list, if not set
        """
        if self.choices is None:
            self.choices = []
            for unit in self.units:
                self.choices.append((unit.id, unit.abbrev, ))

    def get_base_unit(self):
        """
        returns the unit with factor == 1.0
        """
        if self.units:
            for unit in self.units:
                if unit.factor == 1.0:
                    return unit
        return None

    def get_base_unit_id(self):
        """
        returns the identifier of the base unit (e.g. "m²")
        """
        base_unit = self.get_base_unit()
        if base_unit is None:
            return None

        return base_unit.id

    def __init__(self, *args, **kwargs):
        if 'choices' in kwargs:
            raise TypeError("%s invalid attribute 'choices'" % (
                self.__class__.__name__, ))
        self.auto_convert = kwargs.pop('auto_convert', True)
        self.verbose_name = kwargs.get('verbose_name')
        self.blank = kwargs.get('blank')
        self.null = kwargs.get('null')
        self.validators = kwargs.get('validators', [])
        kwargs['editable'] = False
        kwargs['default'] = 0.

        self.default_unit = kwargs.pop('default_unit',
            self.get_base_unit_id())
        self.update_choices()
        super(UnitField, self).__init__(*args, **kwargs)

    def contribute_to_class(self, cls, name):
        self.name = name

        self.input_field = UnitInputField(default=0,
            blank=self.blank,
            null=self.null,
            auto_convert=self.auto_convert,
            validators=self.validators,
            verbose_name=self.verbose_name)
        cls.add_to_class("%s_input" % (self.name,), self.input_field)

        # self.unit_field = CharField(default=self.default_unit, choices=self.choices)
        self.unit_field = ModelCharField(max_length=10,
            default=self.default_unit,
            choices=self.choices)
        cls.add_to_class("%s_unit" % (self.name,), self.unit_field)

        self.value_field = CalculatedFloatField(default=0.0,
            units=self.units,
            blank=self.blank,
            null=self.null)
        cls.add_to_class("%s_value" % (self.name,), self.value_field)

        self.key = md5_hexdigest(self.name)

        field = UnitValueCreator(self)

        setattr(cls, name, field)

    def get_db_prep_save(self, value):
        pass

    def get_db_prep_lookup(self, lookup_type, value):
        raise NotImplementedError(self.get_db_prep_lookup)

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.UnitField}
        defaults.update(kwargs)
        return super(UnitField, self).formfield(**defaults)

# Fields for Base Units

class LengthField(UnitField):
    units = UNITS_LENGTH

class SquareMeasureField(UnitField):
    units = UNITS_SQUARE_MEASURE

class SolidMeasureField(UnitField):
    units = UNITS_SOLID_MEASURE

class MassField(UnitField):
    units = UNITS_MASS

class TimeField(UnitField):
    units = UNITS_TIME

class TemperatureField(UnitField):
    units = UNITS_TEMPERATURE

class AmountOfSubstanceField(UnitField):
    units = UNITS_AMOUNT_OF_SUBSTANCE

class LuminousIntensityField(UnitField):
    units = UNITS_LUMINOUS_INTENSITY

# Fields for Derived Units

class AccelerationField(UnitField):
    units = UNITS_ACCELERATION

class AngleField(UnitField):
    units = UNITS_ANGLE

class CrackleField(UnitField):
    units = UNITS_CRACKLE

class CurrentField(UnitField):
    units = UNITS_CURRENT

class DensityField(UnitField):
    units = UNITS_DENSITY

class ForceField(UnitField):
    units = UNITS_FORCE

class InertiaTorqueField(UnitField):
    units = UNITS_INERTIA_TORQUE

class JerkField(UnitField):
    units = UNITS_JERK

class PotentialField(UnitField):
    units = UNITS_POTENTIAL

class SnapField(UnitField):
    units = UNITS_SNAP

class SpeedField(UnitField):
    units = UNITS_SPEED

class TorqueField(UnitField):
    units = UNITS_TORQUE

class VelocityField(UnitField):
    units = UNITS_VELOCITY

class TorsionField(UnitField):
    units = UNITS_TORSION

class PowerField(UnitField):
    units = UNITS_POWER

class ThermalResistanceField(UnitField):
    units = UNITS_THERMAL_RESISTANCE

class ElectricalTimeConstantField(UnitField):
    units = UNITS_ELECTRICAL_TIME_CONSTANT

class MotorConstantField(UnitField):
    units = UNITS_MOTOR_CONSTANT

class ForceConstantField(UnitField):
    units = UNITS_FORCE_CONSTANT

class PotentialConstantField(UnitField):
    units = UNITS_POTENTIAL_CONSTANT

class ResistanceField(UnitField):
    units = UNITS_ELECTRICAL_RESISTANCE

class InductanceField(UnitField):
    units = UNITS_INDUCTANCE

try:
    from south.modelsinspector import add_introspection_rules
    rules = [((CalculatedFloatField, ), [], {}),]
    add_introspection_rules(rules, [
        "^unit_field\.fields"])
except ImportError:
    pass
<|endoftext|>