<commit_before>'''create training and test samples from the transactions
* add certain fields
* select transactions that contain "reasonable" values

INPUT FILES
 INPUT/transactions-al-g-sfr.csv

OUTPUT FILES
 WORKING/samples-test.csv
 WORKING/samples-train.csv
 WORKING/samples-train-validate.csv
 WORKING/samples-validate.csv
'''

from __future__ import division

import cPickle as pickle
import datetime
import numpy as np
import pandas as pd
import pdb
from pprint import pprint
import random
from sklearn import cross_validation
import sys

from Bunch import Bunch
from columns_contain import columns_contain
from Features import Features
from Logger import Logger
from ParseCommandLine import ParseCommandLine
from Path import Path
import layout_transactions as layout
cc = columns_contain

def usage(msg=None):
    if msg is not None:
        print msg
    print 'usage  : python samples.py [--test]'
    print ' --test: run in test mode'
    sys.exit(1)

def make_control(argv):
    # return a Bunch

    print argv
    if len(argv) not in (1, 2):
        usage('invalid number of arguments')

    pcl = ParseCommandLine(argv)
    arg = Bunch(
        base_name=argv[0].split('.')[0],
        test=pcl.has_arg('--test'),
    )

    random_seed = 123
    random.seed(random_seed)

    dir_working = Path().dir_working()

    debug = False

    out_file_name_base = ('testing-' if arg.test else '') + arg.base_name

    return Bunch(
        arg=arg,
        debug=debug,
        fraction_test=0.2,
        max_sale_price=85e6,  # according to Wall Street Journal
        path_in=dir_working + 'transactions-al-g-sfr.csv',
        path_out_info_reasonable=dir_working + out_file_name_base + '-info-reasonable.pickle',
        path_out_test=dir_working + out_file_name_base + '-test.csv',
        path_out_train=dir_working + out_file_name_base + '-train.csv',
        path_out_train_validate=dir_working + out_file_name_base + '-train-validate.csv',
        path_out_validate=dir_working + out_file_name_base + '-validate.csv',
        random_seed=random_seed,
        test=arg.test,
    )

def report_and_remove(df, keep_masks):
    'return new dataframe with just the kept rows AND info in the table that is printed'
    print 'impact of individual masks'
    format = '%40s removed %6d samples (%3d%%)'
    info = {}
    sorted_names = sorted([name for name in keep_masks.keys()])
    for name in sorted_names:
        keep_mask = keep_masks[name]
        n_removed = len(df) - sum(keep_mask)
        fraction_removed = n_removed / len(df)
        print format % (name, n_removed, 100.0 * fraction_removed)
        info[name] = fraction_removed

    mm = reduce(lambda a, b: a & b, keep_masks.values())
    total_removed = len(df) - sum(mm)
    total_fraction_removed = total_removed / len(df)
    print format % ('*** in combination', total_removed, 100.0 * total_fraction_removed)

    r = df[mm]
    return r, info

def check_never_missing(df, feature_names):
    'verify that each ege feature is always present'
    print
    print 'Each of these fields should never be missing'
    total_missing = 0
    format_string = 'field %40s is missing %7d times'
    for name in feature_names:
        count_missing = sum(pd.isnull(df[name]))
        print format_string % (name, count_missing)
        total_missing += count_missing
    print format_string % ('** total across fields **', total_missing)
    print 'total missing', total_missing
    if total_missing > 0:
        pdb.set_trace()  # should not be any missing
        pass

def check_no_zeros(df, feature_names):
    'check that only a few expected fields have zero values'
    # TODO: some of the features can have zero values!
    total_zeros = 0
    format_string = 'field %40s is zero %7d times'
    for name in feature_names:
        if ('_has_' in name) or ('is_' in name):
            continue  # these are 0/1 indicator features
        if name in (layout.building_bedrooms,
                    layout.building_basement_square_feet,
                    layout.building_fireplace_number,
                    layout.census2000_fraction_owner_occupied,
                    layout.has_pool,
                    layout.parking_spaces,):
            continue  # also, these should be often zero
        count_zero = sum(df[name] == 0)
        print format_string % (name, count_zero)
        total_zeros += count_zero
    print format_string % ('** total across fields **', total_zeros)
    if total_zeros > 0:
        print 'found some unexpected zero values'
        pdb.set_trace()  # should not be any zeros

def check_feature_values(df):
    # age features are added just before a model is fitted
    feature_names = sorted([x for x, y in Features().ege() if 'age' not in x])
    check_never_missing(df, feature_names)
    check_no_zeros(df, feature_names)

def reasonable_feature_values(df, control):
    'return new DataFrame containing sample in df that have "reasonable" values'
    def below(percentile, series):
        quantile_value = series.quantile(percentile / 100.0)
        r = series < quantile_value
        return r

    # set mask value in m to True to keep the observation
    m = {}
    m['assessment total > 0'] = df[layout.assessment_total] > 0
    m['assessment land > 0'] = df[layout.assessment_land] > 0
    m['assessment improvement > 0'] = df[layout.assessment_improvement] > 0
    m['baths > 0'] = df[layout.building_baths] > 0
    m['effective year built >= year built'] = df[layout.year_built_effective] >= df[layout.year_built]
    m['full price'] = layout.mask_full_price(df)
    m['latitude known'] = layout.mask_gps_latitude_known(df)
    m['longitude known'] = layout.mask_gps_longitude_known(df)
    m['land size < 99th percentile'] = below(99, df[layout.lot_land_square_feet])
    m['land size > 0'] = df[layout.lot_land_square_feet] > 0
    m['living size < 99th percentile'] = below(99, df[layout.building_living_square_feet])
    m['living square feet > 0'] = df[layout.building_living_square_feet] > 0
    m['median household income > 0'] = df[layout.census2000_median_household_income] > 0
    m['new or resale'] = layout.mask_new_or_resale(df)
    m['one building'] = layout.mask_is_one_building(df)
    m['one APN'] = layout.mask_is_one_parcel(df)
    # m['recording date present'] = ~df[layout.recording_date + '_deed'].isnull()  # ~ => not
    m['price > 0'] = df[layout.price] > 0
    m['price < max'] = df[layout.price] < control.max_sale_price
    m['rooms > 0'] = df[layout.building_rooms] > 0
    m['resale or new construction'] = (
        layout.mask_is_new_construction(df) |
        layout.mask_is_resale(df)
    )
    m['sale date present'] = layout.mask_sale_date_present(df)
    m['sale date valid'] = layout.mask_sale_date_valid(df)
    m['stories > 0'] = df[layout.building_stories] > 0
    m['units == 1'] = df[layout.n_units] == 1
    m['year_built > 0'] = df[layout.year_built] > 0

    print 'effects of reasonable values'
    return report_and_remove(df, m)

def add_features(df, control):
    '''mutate df

    NOTE: adding features from the 2009 taxroll leaks the future into the transaction
    set. However, the damage is perhaps small because most properties do not change
    their character: a single family house in 2009 was very likely a single family
    house in 2003
    '''
    def split(date):
        year = int(date / 10000)
        md = date - year * 10000
        month = int(md / 100)
        day = md - month * 100
        return year, month, day

    def python_date(date):
        'yyyymmdd --> datetime.date(year, month, day)'
        year, month, day = split(date)
        return datetime.date(int(year), int(month), int(day))

    def yyyymm(date):
        year, month, day = split(date)
        return year * 100 + month

    # create sale-date related features
    def append_column(name, values):
        df.insert(len(df.columns),
                  name,
                  pd.Series(values, index=df.index),
                  )

    sale_date = df[layout.sale_date]
    append_column(layout.sale_date_python, sale_date.apply(python_date))
    append_column(layout.yyyymm, sale_date.apply(yyyymm))

    # create indicator features:w
    append_column(layout.is_new_construction, layout.mask_is_new_construction(df))
    append_column(layout.is_resale, layout.mask_is_resale(df))
    append_column(layout.building_has_basement, df[layout.building_basement_square_feet] > 0)
    append_column(layout.building_has_fireplace, df[layout.building_fireplace_number] > 0)
    append_column(layout.has_parking, df[layout.parking_spaces] > 0)
    append_column(layout.has_pool, df[layout.pool_flag] == 'Y')

    # create additional indicators aggregating certain PROPN codes
    def create(new_column_base, ored_column_bases):

        def create2(prefix):
            def ored_name(ored_index):
                return prefix + '_has_' + ored_column_bases[ored_index]

            mask = df[ored_name(0)]
            for index in range(1, len(ored_column_bases)):
                mask2 = df[ored_name(index)]
                mask = mask | mask2
            append_column(prefix + '_has_' + new_column_base, mask)

        for prefix in ('census_tract', 'zip5'):
            create2(prefix)

    create('any_commercial', ('commercial', 'commercial_condominium',))
    create('any_industrial', ('industrial', 'industrial_light', 'industrial_heavy',))
    create('any_non_residential', ('amusement', 'any_commercial', 'financial_institution', 'hotel',
                                   'any_industrial', 'medical', 'office_building', 'parking',
                                   'retail', 'service', 'transport', 'utilities', 'warehouse',))

def main(argv):
    control = make_control(argv)
    sys.stdout = Logger(base_name=control.arg.base_name)
    print control

    transactions = pd.read_csv(control.path_in,
                               nrows=100000 if control.arg.test else None,
                               )
    print 'transactions shape', transactions.shape

    after_2000_census_known = transactions[layout.mask_sold_after_2002(transactions)]
    print 'after 2000 census known shape', after_2000_census_known.shape

    subset, info_reasonable = reasonable_feature_values(after_2000_census_known, control)
    print 'subset shape', subset.shape

    add_features(subset, control)
    check_feature_values(subset)

    # split into test and train
    # stratify by yyyymm (month of sale)

    def count_yyyymm(df, yyyymm):
        return sum(df[layout.yyyymm] == yyyymm)

    def split(df, fraction_test):
        sss = cross_validation.StratifiedShuffleSplit(
            y=df.yyyymm,
            n_iter=1,
            test_size=fraction_test,
            train_size=None,
            random_state=control.random_seed,
        )
        assert len(sss) == 1
        for train_index, test_index in sss:
            train = df.iloc[train_index]
            test = df.iloc[test_index]
        return test, train

    # split samples into test and train, stratified on month of sale
    # then split training data on test (validate) and train
    test, train = split(subset, control.fraction_test)
    fraction_test = control.fraction_test / (1 - control.fraction_test)
    train_test, train_train = split(train, fraction_test)

    # write the csv files
    test.to_csv(control.path_out_test)
    train.to_csv(control.path_out_train)
    train_test.to_csv(control.path_out_train_validate)
    train_train.to_csv(control.path_out_validate)

    # count samples in each strata (= month)
    yyyymms = sorted(set(subset[layout.yyyymm]))
    format_string = '%6d # total %6d # test %6d # train %6d # train_test %6d # train_train %6d'
    for yyyymm in yyyymms:
        c1 = count_yyyymm(subset, yyyymm)
        c2 = count_yyyymm(test, yyyymm)
        c3 = count_yyyymm(train, yyyymm)
        c4 = count_yyyymm(train_test, yyyymm)
        c5 = count_yyyymm(train_train, yyyymm)
        print format_string % (yyyymm, c1, c2, c3, c4, c5)
        if c1 != c2 + c3:
            print 'c1 not exactly split'
            pdb.set_trace()
        if c3 != c4 + c5:
            print 'c3 not exactly split'
            pdb.set_trace()
    print 'totals'
    print format_string % (0, len(subset), len(test), len(train), len(train_train), len(train_test))

    f = open(control.path_out_info_reasonable, 'wb')
    pickle.dump((info_reasonable, control), f)
    f.close()

    print control
    if control.test:
        print 'DISCARD OUTPUT: test'
    print 'done'

    return

if __name__ == '__main__':
    if False:
        # avoid pyflakes warnings
        pdb.set_trace()
        pprint()
        pd.DataFrame()
        np.array()

    main(sys.argv)
<commit_msg>create age-related fields<commit_after>'''create training and test samples from the transactions
* add certain fields
* select transactions that contain "reasonable" values

INPUT FILES
 INPUT/transactions-al-g-sfr.csv

OUTPUT FILES
 WORKING/samples-test.csv
 WORKING/samples-train.csv
 WORKING/samples-train-validate.csv
 WORKING/samples-validate.csv
'''

from __future__ import division

import cPickle as pickle
import datetime
import numpy as np
import pandas as pd
import pdb
from pprint import pprint
import random
from sklearn import cross_validation
import sys

from Bunch import Bunch
from columns_contain import columns_contain
from Features import Features
from Logger import Logger
from ParseCommandLine import ParseCommandLine
from Path import Path
import layout_transactions as layout
cc = columns_contain

def usage(msg=None):
    if msg is not None:
        print msg
    print 'usage  : python samples.py [--test]'
    print ' --test: run in test mode'
    sys.exit(1)

def make_control(argv):
    # return a Bunch

    print argv
    if len(argv) not in (1, 2):
        usage('invalid number of arguments')

    pcl = ParseCommandLine(argv)
    arg = Bunch(
        base_name=argv[0].split('.')[0],
        test=pcl.has_arg('--test'),
    )

    random_seed = 123
    random.seed(random_seed)

    dir_working = Path().dir_working()

    debug = False

    out_file_name_base = ('testing-' if arg.test else '') + arg.base_name

    return Bunch(
        arg=arg,
        debug=debug,
        fraction_test=0.2,
        max_sale_price=85e6,  # according to Wall Street Journal
        path_in=dir_working + 'transactions-al-g-sfr.csv',
        path_out_info_reasonable=dir_working + out_file_name_base + '-info-reasonable.pickle',
        path_out_test=dir_working + out_file_name_base + '-test.csv',
        path_out_train=dir_working + out_file_name_base + '-train.csv',
        path_out_train_validate=dir_working + out_file_name_base + '-train-validate.csv',
        path_out_validate=dir_working + out_file_name_base + '-validate.csv',
        random_seed=random_seed,
        test=arg.test,
    )

def report_and_remove(df, keep_masks):
    'return new dataframe with just the kept rows AND info in the table that is printed'
    print 'impact of individual masks'
    format = '%40s removed %6d samples (%3d%%)'
    info = {}
    sorted_names = sorted([name for name in keep_masks.keys()])
    for name in sorted_names:
        keep_mask = keep_masks[name]
        n_removed = len(df) - sum(keep_mask)
        fraction_removed = n_removed / len(df)
        print format % (name, n_removed, 100.0 * fraction_removed)
        info[name] = fraction_removed

    mm = reduce(lambda a, b: a & b, keep_masks.values())
    total_removed = len(df) - sum(mm)
    total_fraction_removed = total_removed / len(df)
    print format % ('*** in combination', total_removed, 100.0 * total_fraction_removed)

    r = df[mm]
    return r, info

def check_never_missing(df, feature_names):
    'verify that each ege feature is always present'
    print
    print 'Each of these fields should never be missing'
    total_missing = 0
    format_string = 'field %40s is missing %7d times'
    for name in feature_names:
        count_missing = sum(pd.isnull(df[name]))
        print format_string % (name, count_missing)
        total_missing += count_missing
    print format_string % ('** total across fields **', total_missing)
    print 'total missing', total_missing
    if total_missing > 0:
        pdb.set_trace()  # should not be any missing
        pass

def check_no_zeros(df, feature_names):
    'check that only a few expected fields have zero values'
    # TODO: some of the features can have zero values!
    total_zeros = 0
    format_string = 'field %40s is zero %7d times'
    for name in feature_names:
        if ('_has_' in name) or ('is_' in name):
            continue  # these are 0/1 indicator features
        if name in (layout.building_bedrooms,
                    layout.building_basement_square_feet,
                    layout.building_fireplace_number,
                    layout.census2000_fraction_owner_occupied,
                    layout.has_pool,
                    layout.parking_spaces,):
            continue  # also, these should be often zero
        count_zero = sum(df[name] == 0)
        print format_string % (name, count_zero)
        total_zeros += count_zero
    print format_string % ('** total across fields **', total_zeros)
    if total_zeros > 0:
        print 'found some unexpected zero values'
        pdb.set_trace()  # should not be any zeros

def check_feature_values(df):
    # age features are added just before a model is fitted
    feature_names = sorted([x for x, y in Features().ege() if 'age' not in x])
    check_never_missing(df, feature_names)
    check_no_zeros(df, feature_names)

def reasonable_feature_values(df, control):
    'return new DataFrame containing sample in df that have "reasonable" values'
    def below(percentile, series):
        quantile_value = series.quantile(percentile / 100.0)
        r = series < quantile_value
        return r

    # set mask value in m to True to keep the observation
    m = {}
    m['assessment total > 0'] = df[layout.assessment_total] > 0
    m['assessment land > 0'] = df[layout.assessment_land] > 0
    m['assessment improvement > 0'] = df[layout.assessment_improvement] > 0
    m['baths > 0'] = df[layout.building_baths] > 0
    m['effective year built >= year built'] = df[layout.year_built_effective] >= df[layout.year_built]
    m['full price'] = layout.mask_full_price(df)
    m['latitude known'] = layout.mask_gps_latitude_known(df)
    m['longitude known'] = layout.mask_gps_longitude_known(df)
    m['land size < 99th percentile'] = below(99, df[layout.lot_land_square_feet])
    m['land size > 0'] = df[layout.lot_land_square_feet] > 0
    m['living size < 99th percentile'] = below(99, df[layout.building_living_square_feet])
    m['living square feet > 0'] = df[layout.building_living_square_feet] > 0
    m['median household income > 0'] = df[layout.census2000_median_household_income] > 0
    m['new or resale'] = layout.mask_new_or_resale(df)
    m['one building'] = layout.mask_is_one_building(df)
    m['one APN'] = layout.mask_is_one_parcel(df)
    # m['recording date present'] = ~df[layout.recording_date + '_deed'].isnull()  # ~ => not
    m['price > 0'] = df[layout.price] > 0
    m['price < max'] = df[layout.price] < control.max_sale_price
    m['rooms > 0'] = df[layout.building_rooms] > 0
    m['resale or new construction'] = (
        layout.mask_is_new_construction(df) |
        layout.mask_is_resale(df)
    )
    m['sale date present'] = layout.mask_sale_date_present(df)
    m['sale date valid'] = layout.mask_sale_date_valid(df)
    m['stories > 0'] = df[layout.building_stories] > 0
    m['units == 1'] = df[layout.n_units] == 1
    m['year_built > 0'] = df[layout.year_built] > 0

    print 'effects of reasonable values'
    return report_and_remove(df, m)

def add_features(df, control):
    def split(date):
        year = int(date / 10000)
        md = date - year * 10000
        month = int(md / 100)
        day = md - month * 100
        return year, month, day

    def python_date(date):
        'yyyymmdd --> datetime.date(year, month, day)'
        year, month, day = split(date)
        return datetime.date(int(year), int(month), int(day))

    def year(date):
        'yyyymmdd --> datetime.date(year, 7, 1); mid point of year'
        year, month, day = split(date)
        return year

    def yyyymm(date):
        year, month, day = split(date)
        return year * 100 + month

    # create sale-date related features
    def append_column(name, values):
        df.insert(len(df.columns),
                  name,
                  pd.Series(values, index=df.index),
                  )

    sale_date = df[layout.sale_date]
    sale_date_python = sale_date.apply(python_date)
    append_column(layout.sale_date_python, sale_date_python)
    append_column(layout.yyyymm, sale_date.apply(yyyymm))

    # create age and similar fields
    # NOTE: these are ages at the sale date
    sale_year = sale_date.apply(year)
    year_built = df[layout.year_built]
    effective_year_built = df[layout.year_built_effective]
    age = sale_year - year_built
    effective_age = sale_year - effective_year_built
    append_column(layout.age, age)
    append_column(layout.age2, age * age)
    append_column(layout.age_effective, effective_age)
    append_column(layout.age_effective2, effective_age * effective_age)

    # create indicator features
    append_column(layout.is_new_construction, layout.mask_is_new_construction(df))
    append_column(layout.is_resale, layout.mask_is_resale(df))
    append_column(layout.building_has_basement, df[layout.building_basement_square_feet] > 0)
    append_column(layout.building_has_fireplace, df[layout.building_fireplace_number] > 0)
    append_column(layout.has_parking, df[layout.parking_spaces] > 0)
    append_column(layout.has_pool, df[layout.pool_flag] == 'Y')

    # create additional indicators aggregating certain PROPN codes
    def create(new_column_base, ored_column_bases):

        def create2(prefix):
            def ored_name(ored_index):
                return prefix + '_has_' + ored_column_bases[ored_index]

            mask = df[ored_name(0)]
            for index in range(1, len(ored_column_bases)):
                mask2 = df[ored_name(index)]
                mask = mask | mask2
            append_column(prefix + '_has_' + new_column_base, mask)

        for prefix in ('census_tract', 'zip5'):
            create2(prefix)

    create('any_commercial', ('commercial', 'commercial_condominium',))
    create('any_industrial', ('industrial', 'industrial_light', 'industrial_heavy',))
    create('any_non_residential', ('amusement', 'any_commercial', 'financial_institution', 'hotel',
                                   'any_industrial', 'medical', 'office_building', 'parking',
                                   'retail', 'service', 'transport', 'utilities', 'warehouse',))

def main(argv):
    control = make_control(argv)
    sys.stdout = Logger(base_name=control.arg.base_name)
    print control

    transactions = pd.read_csv(control.path_in,
                               nrows=100000 if control.arg.test else None,
                               )
    print 'transactions shape', transactions.shape

    after_2000_census_known = transactions[layout.mask_sold_after_2002(transactions)]
    print 'after 2000 census known shape', after_2000_census_known.shape

    subset, info_reasonable = reasonable_feature_values(after_2000_census_known, control)
    print 'subset shape', subset.shape

    add_features(subset, control)
    check_feature_values(subset)

    # split into test and train
    # stratify by yyyymm (month of sale)

    def count_yyyymm(df, yyyymm):
        return sum(df[layout.yyyymm] == yyyymm)

    def split(df, fraction_test):
        sss = cross_validation.StratifiedShuffleSplit(
            y=df.yyyymm,
            n_iter=1,
            test_size=fraction_test,
            train_size=None,
            random_state=control.random_seed,
        )
        assert len(sss) == 1
        for train_index, test_index in sss:
            train = df.iloc[train_index]
            test = df.iloc[test_index]
        return test, train

    # split samples into test and train, stratified on month of sale
    # then split training data on test (validate) and train
    test, train = split(subset, control.fraction_test)
    fraction_test = control.fraction_test / (1 - control.fraction_test)
    train_test, train_train = split(train, fraction_test)

    # write the csv files
    test.to_csv(control.path_out_test)
    train.to_csv(control.path_out_train)
    train_test.to_csv(control.path_out_train_validate)
    train_train.to_csv(control.path_out_validate)

    # count samples in each strata (= month)
    yyyymms = sorted(set(subset[layout.yyyymm]))
    format_string = '%6d # total %6d # test %6d # train %6d # train_test %6d # train_train %6d'
    for yyyymm in yyyymms:
        c1 = count_yyyymm(subset, yyyymm)
        c2 = count_yyyymm(test, yyyymm)
        c3 = count_yyyymm(train, yyyymm)
        c4 = count_yyyymm(train_test, yyyymm)
        c5 = count_yyyymm(train_train, yyyymm)
        print format_string % (yyyymm, c1, c2, c3, c4, c5)
        if c1 != c2 + c3:
            print 'c1 not exactly split'
            pdb.set_trace()
        if c3 != c4 + c5:
            print 'c3 not exactly split'
            pdb.set_trace()
    print 'totals'
    print format_string % (0, len(subset), len(test), len(train), len(train_train), len(train_test))

    f = open(control.path_out_info_reasonable, 'wb')
    pickle.dump((info_reasonable, control), f)
    f.close()

    print control
    if control.test:
        print 'DISCARD OUTPUT: test'
    print 'done'

    return

if __name__ == '__main__':
    if False:
        # avoid pyflakes warnings
        pdb.set_trace()
        pprint()
        pd.DataFrame()
        np.array()

    main(sys.argv)
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright (C) 2009 Itaapy, ArsAperta, Pierlis, Talend

# Import from the Standard Library
from datetime import datetime, timedelta
from unittest import TestCase, main

# Import from the XML Library
from lxml.etree import _Element

# Import from lpod
from lpod.document import odf_get_document
from lpod.utils import _make_xpath_query, _check_arguments
from lpod.utils import DateTime, Duration, Boolean
from lpod.xmlpart import odf_create_element

class GenerateXPathTestCase(TestCase):

    def test_element(self):
        query = _make_xpath_query('text:p')
        self.assertEqual(query, '//text:p')

    def test_attribute(self):
        query = _make_xpath_query('text:p', style='Standard')
        self.assertEqual(query, '//text:p[@text:style-name="Standard"]')

    def test_two_attributes(self):
        query = _make_xpath_query('text:h', style='Standard', level=1)
        expected = ('//text:h[@text:outline-level="1"]'
                    '[@text:style-name="Standard"]')
        self.assertEqual(query, expected)

    def test_position(self):
        query = _make_xpath_query('text:h', position=2)
        self.assertEqual(query, '//text:h[2]')

    def test_attribute_position(self):
        query = _make_xpath_query('text:p', style='Standard', position=2)
        self.assertEqual(query, '//text:p[@text:style-name="Standard"][2]')

    def test_two_attributes_position(self):
        query = _make_xpath_query('text:h', style='Standard', level=1,
                                  position=2)
        expected = ('//text:h[@text:outline-level="1"]'
                    '[@text:style-name="Standard"][2]')
        self.assertEqual(query, expected)

class CheckArgumentsTestCase(TestCase):

    def setUp(self):
        self.document = odf_get_document('samples/example.odt')

    def tearDown(self):
        del self.document

    def test_bad_context(self):
        document = self.document
        self.assertRaises(TypeError, _check_arguments, context=document)

    def test_bad_element(self):
        self.assertRaises(TypeError, _check_arguments, element=_Element)

    def test_str_xmlposition(self):
        self.assertRaises(ValueError, _check_arguments, xmlposition='after')

    def test_big_xmlposition(self):
        self.assertRaises(ValueError, _check_arguments, xmlposition=999)

    def test_str_position(self):
        self.assertRaises(TypeError, _check_arguments, position='1')

    def test_position_zero(self):
        self.assertRaises(ValueError, _check_arguments, position=0)

    def test_str_level(self):
        self.assertRaises(TypeError, _check_arguments, level='1')

    def test_level_zero(self):
        self.assertRaises(ValueError, _check_arguments, level=0)

    def test_bad_text(self):
        self.assertRaises(TypeError, _check_arguments, text='Hello')

    def test_bad_style(self):
        data = ('<style:master-page '
                'style:name="Standard" '
                'style:page-layout-name="Mpm1"/>')
        element = odf_create_element(data)
        self.assertRaises(TypeError, _check_arguments, style=element)

    def test_bad_family(self):
        self.assertRaises(ValueError, _check_arguments, family='heading')

    def test_bad_cell_type(self):
        self.assertRaises(ValueError, _check_arguments, cell_type='integer')

    def test_missing_currency(self):
        self.assertRaises(ValueError, _check_arguments, cell_type='currency')

    def test_bad_author(self):
        self.assertRaises(TypeError, _check_arguments, author='Plato')

    def test_bad_date(self):
        self.assertRaises(TypeError, _check_arguments,
                          date='2009-06-22T17:18:42')

    def test_bad_start_date(self):
        self.assertRaises(TypeError, _check_arguments,
                          start_date='2009-06-22T17:18:42')

    def test_bad_end_date(self):
        self.assertRaises(TypeError, _check_arguments,
                          end_date='2009-06-22T17:18:42')

    def test_bad_offset(self):
        self.assertRaises(TypeError, _check_arguments, offset='a word')

    def test_position_name(self):
        document = self.document
        content = document.get_xmlpart('content')
        self.assertRaises(ValueError, content.get_frame, position=1,
                          name='foo')
        self.assertRaises(ValueError, content.get_frame, position=None,
                          name=None)

class DateTimeTestCase(TestCase):

    def test_encode(self):
        date = datetime(2009, 06, 26, 11, 9, 36)
        expected = '2009-06-26T11:09:36'
        self.assertEqual(DateTime.encode(date), expected)

    def test_decode(self):
        date = '2009-06-29T14:33:21'
        expected = datetime(2009, 6, 29, 14, 33, 21)
        self.assertEqual(DateTime.decode(date), expected)

class DurationTestCase(TestCase):

    def test_encode(self):
        duration = timedelta(0, 53, 0, 0, 6)
        expected = 'PT00H06M53S'
        self.assertEqual(Duration.encode(duration), expected)

    def test_decode(self):
        duration = 'PT12H34M56S'
        expected = timedelta(0, 56, 0, 0, 34, 12)
        self.assertEqual(Duration.decode(duration), expected)

class BooleanTestCase(TestCase):

    def test_encode(self):
        self.assertEqual(Boolean.encode(True), 'true')
        self.assertEqual(Boolean.encode(False), 'false')

    def test_bad_encode(self):
        self.assertRaises(TypeError, Boolean.encode, 'true')
        self.assertRaises(TypeError, Boolean.encode, 1)

    def test_decode(self):
        self.assertEqual(Boolean.decode('true'), True)
        self.assertEqual(Boolean.decode('false'), False)

    def test_bad_decode(self):
        self.assertRaises(ValueError, Boolean.decode, 'True')
        self.assertRaises(ValueError, Boolean.decode, '1')

if __name__ == '__main__':
    main()
<commit_msg>python: test: Fix a bug with cell_type / value_type<commit_after># -*- coding: UTF-8 -*-
# Copyright (C) 2009 Itaapy, ArsAperta, Pierlis, Talend

# Import from the Standard Library
from datetime import datetime, timedelta
from unittest import TestCase, main

# Import from the XML Library
from lxml.etree import _Element

# Import from lpod
from lpod.document import odf_get_document
from lpod.utils import _make_xpath_query, _check_arguments
from lpod.utils import DateTime, Duration, Boolean
from lpod.xmlpart import odf_create_element

class GenerateXPathTestCase(TestCase):

    def test_element(self):
        query = _make_xpath_query('text:p')
        self.assertEqual(query, '//text:p')

    def test_attribute(self):
        query = _make_xpath_query('text:p', style='Standard')
        self.assertEqual(query, '//text:p[@text:style-name="Standard"]')

    def test_two_attributes(self):
        query = _make_xpath_query('text:h', style='Standard', level=1)
        expected = ('//text:h[@text:outline-level="1"]'
                    '[@text:style-name="Standard"]')
        self.assertEqual(query, expected)

    def test_position(self):
        query = _make_xpath_query('text:h', position=2)
        self.assertEqual(query, '//text:h[2]')

    def test_attribute_position(self):
        query = _make_xpath_query('text:p', style='Standard', position=2)
        self.assertEqual(query, '//text:p[@text:style-name="Standard"][2]')

    def test_two_attributes_position(self):
        query = _make_xpath_query('text:h', style='Standard', level=1,
                                  position=2)
        expected = ('//text:h[@text:outline-level="1"]'
                    '[@text:style-name="Standard"][2]')
        self.assertEqual(query, expected)

class CheckArgumentsTestCase(TestCase):

    def setUp(self):
        self.document = odf_get_document('samples/example.odt')

    def tearDown(self):
        del self.document

    def test_bad_context(self):
        document = self.document
        self.assertRaises(TypeError, _check_arguments, context=document)

    def test_bad_element(self):
        self.assertRaises(TypeError, _check_arguments, element=_Element)

    def test_str_xmlposition(self):
        self.assertRaises(ValueError, _check_arguments, xmlposition='after')

    def test_big_xmlposition(self):
        self.assertRaises(ValueError, _check_arguments, xmlposition=999)

    def test_str_position(self):
        self.assertRaises(TypeError, _check_arguments, position='1')

    def test_position_zero(self):
        self.assertRaises(ValueError, _check_arguments, position=0)

    def test_str_level(self):
        self.assertRaises(TypeError, _check_arguments, level='1')

    def test_level_zero(self):
        self.assertRaises(ValueError, _check_arguments, level=0)

    def test_bad_text(self):
        self.assertRaises(TypeError, _check_arguments, text='Hello')

    def test_bad_style(self):
        data = ('<style:master-page '
                'style:name="Standard" '
                'style:page-layout-name="Mpm1"/>')
        element = odf_create_element(data)
        self.assertRaises(TypeError, _check_arguments, style=element)

    def test_bad_family(self):
        self.assertRaises(ValueError, _check_arguments, family='heading')

    def test_bad_cell_type(self):
        self.assertRaises(ValueError, _check_arguments, value_type='integer')

    def test_missing_currency(self):
        self.assertRaises(ValueError, _check_arguments, value_type='currency')

    def test_bad_author(self):
        self.assertRaises(TypeError, _check_arguments, author='Plato')

    def test_bad_date(self):
        self.assertRaises(TypeError, _check_arguments,
                          date='2009-06-22T17:18:42')

    def test_bad_start_date(self):
        self.assertRaises(TypeError, _check_arguments,
                          start_date='2009-06-22T17:18:42')

    def test_bad_end_date(self):
        self.assertRaises(TypeError, _check_arguments,
                          end_date='2009-06-22T17:18:42')

    def test_bad_offset(self):
        self.assertRaises(TypeError, _check_arguments, offset='a word')

    def test_position_name(self):
        document = self.document
        content = document.get_xmlpart('content')
        self.assertRaises(ValueError, content.get_frame, position=1,
                          name='foo')
        self.assertRaises(ValueError, content.get_frame, position=None,
                          name=None)

class DateTimeTestCase(TestCase):

    def test_encode(self):
        date = datetime(2009, 06, 26, 11, 9, 36)
        expected = '2009-06-26T11:09:36'
        self.assertEqual(DateTime.encode(date), expected)

    def test_decode(self):
        date = '2009-06-29T14:33:21'
        expected = datetime(2009, 6, 29, 14, 33, 21)
        self.assertEqual(DateTime.decode(date), expected)

class DurationTestCase(TestCase):

    def test_encode(self):
        duration = timedelta(0, 53, 0, 0, 6)
        expected = 'PT00H06M53S'
        self.assertEqual(Duration.encode(duration), expected)

    def test_decode(self):
        duration = 'PT12H34M56S'
        expected = timedelta(0, 56, 0, 0, 34, 12)
        self.assertEqual(Duration.decode(duration), expected)

class BooleanTestCase(TestCase):

    def test_encode(self):
        self.assertEqual(Boolean.encode(True), 'true')
        self.assertEqual(Boolean.encode(False), 'false')

    def test_bad_encode(self):
        self.assertRaises(TypeError, Boolean.encode, 'true')
        self.assertRaises(TypeError, Boolean.encode, 1)

    def test_decode(self):
        self.assertEqual(Boolean.decode('true'), True)
        self.assertEqual(Boolean.decode('false'), False)

    def test_bad_decode(self):
        self.assertRaises(ValueError, Boolean.decode, 'True')
        self.assertRaises(ValueError, Boolean.decode, '1')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Nosetests for tic-tac-toe.py

see http://somethingaboutorange.com/mrl/projects/nose/
"""
import tic_tac_toe

class TestGameBoard(object):

    def setup(self):
        self.board = tic_tac_toe.GameBoard()

    def should_return_a_blank_board_on_initialisation_test(self):
        assert self.board.state == [ '_', '_', '_', '_', '_', '_', '_', '_', '_', ]

    def set_x_on_first_position_test(self):
        self.board.set(0, 0, 'X')
        assert self.board.state == [ 'X', '_', '_', '_', '_', '_', '_', '_', '_', ]

    def set_two_different_numbers_test(self):
        self.board.set(1, 2, 'X')
        assert self.board.state == [ '_', '_', '_', '_', '_', '_', '_', 'X', '_', ]

    def set_two_moves_test(self):
        assert self.board.set(1, 0, 'X')
        assert self.board.set(0, 2, 'O')
        assert self.board.state == [ '_', 'X', '_', '_', '_', '_', 'O', '_', '_', ]

    def attempt_illegal_move_test(self):
        assert self.board.set(0, 0, 'X')
        status = self.board.set(0, 0, 'O')
        assert self.board.state == [ 'X', '_', '_', '_', '_', '_', '_', '_', '_', ]
        assert not status

    def show_board_test(self):
        assert repr(self.board) == "___\n___\n___"

class TestGame(object):

    def setup(self):
        self.game = tic_tac_toe.Game()

    def test_new_game_has_empty_board(self):
        assert self.game.show_board() == """\
___
___
___"""

    def test_1st_move_is_X_test(self):
        self.game.take_turn(0,0)
        assert self.game.show_board() == """\
X__
___
___"""

    def test_2_moves(self):
        assert self.game.take_turn(0,0)
        assert self.game.take_turn(1,1)
        assert self.game.show_board() == """\
X__
_O_
___"""

    def test_invalid_move(self):
        assert self.game.take_turn(0,0)
        board_before = self.game.show_board()
        assert not self.game.take_turn(0,0)
        assert self.game.show_board() == board_before

    def test_winning(self):
        self.game.take_turn(0,0)
        self.game.take_turn(0,1)
        self.game.take_turn(1,0)
        self.game.take_turn(0,2)
        self.game.take_turn(2,0)
        assert self.game.get_winner() == 'X'

def not_yet():
    assert status == 'OK'
    status, state = ttt.play('X', 0)
    assert state == [ 'X', '_', '_', '_', '_', '_', '_', '_', '_', ]
    assert status == 'OK'

def not_yet():
    status, state = ttt.play('O', 5)
    assert state == [ 'X', '_', '_', '_', '_', 'O', '_', '_', '_', ]
    assert status == 'OK'
    status, state = ttt.play('X', 0)
    assert state == [ 'X', '_', '_', '_', '_', 'O', '_', '_', '_', ]
    assert status == 'TAKEN'
    status, state = ttt.play('O', 6)
    assert state == [ 'X', '_', '_', '_', '_', 'O', '_', '_', '_', ]
    assert status == 'BAD TURN'
    status, state = ttt.play('X', 1)
    assert state == [ 'X', 'X', '_', '_', '_', 'O', '_', '_', '_', ]
    assert status == 'OK'
    status, state = ttt.play('O', 6)
    assert state == [ 'X', 'X', '_', '_', '_', 'O', 'O', '_', '_', ]
    assert status == 'OK'
    status, state = ttt.play('X', 2)
    assert state == [ 'X', 'X', 'X', '_', '_', 'O', 'O', '_', '_', ]
    assert status == 'X WINS'
<commit_msg>[Peter and Daniele] winnning pair<commit_after>"""
Nosetests for tic-tac-toe.py

see http://somethingaboutorange.com/mrl/projects/nose/
"""
import tic_tac_toe

class TestGameBoard(object):

    def setup(self):
        self.board = tic_tac_toe.GameBoard()

    def should_return_a_blank_board_on_initialisation_test(self):
        assert self.board.state == [ '_', '_', '_', '_', '_', '_', '_', '_', '_', ]

    def set_x_on_first_position_test(self):
        self.board.set(0, 0, 'X')
        assert self.board.state == [ 'X', '_', '_', '_', '_', '_', '_', '_', '_', ]

    def set_two_different_numbers_test(self):
        self.board.set(1, 2, 'X')
        assert self.board.state == [ '_', '_', '_', '_', '_', '_', '_', 'X', '_', ]

    def set_two_moves_test(self):
        assert self.board.set(1, 0, 'X')
        assert self.board.set(0, 2, 'O')
        assert self.board.state == [ '_', 'X', '_', '_', '_', '_', 'O', '_', '_', ]

    def attempt_illegal_move_test(self):
        assert self.board.set(0, 0, 'X')
        status = self.board.set(0, 0, 'O')
        assert self.board.state == [ 'X', '_', '_', '_', '_', '_', '_', '_', '_', ]
        assert not status

    def show_board_test(self):
        assert repr(self.board) == "___\n___\n___"

class TestGame(object):

    def setup(self):
        self.game = tic_tac_toe.Game()

    def test_new_game_has_empty_board(self):
        assert self.game.show_board() == """\
___
___
___"""

    def test_1st_move_is_X_test(self):
        self.game.take_turn(0,0)
        assert self.game.show_board() == """\
X__
___
___"""

    def test_2_moves(self):
        assert self.game.take_turn(0,0)
        assert self.game.take_turn(1,1)
        assert self.game.show_board() == """\
X__
_O_
___"""

    def test_invalid_move(self):
        assert self.game.take_turn(0,0)
        board_before = self.game.show_board()
        assert not self.game.take_turn(0,0)
        assert self.game.show_board() == board_before

    def test_winning(self):
        assert self.game.get_winner() is None
        assert self.game.take_turn(0,0)
        assert self.game.get_winner() is None
        assert self.game.take_turn(0,1)
        assert self.game.get_winner() is None
        assert self.game.take_turn(1,0)
        assert self.game.get_winner() is None
        assert self.game.take_turn(0,2)
        assert self.game.get_winner() is None
        assert self.game.take_turn(2,0)
        assert self.game.get_winner() == 'X'

def not_yet():
    assert status == 'OK'
    status, state = ttt.play('X', 0)
    assert state == [ 'X', '_', '_', '_', '_', '_', '_', '_', '_', ]
    assert status == 'OK'

def not_yet():
    status, state = ttt.play('O', 5)
    assert state == [ 'X', '_', '_', '_', '_', 'O', '_', '_', '_', ]
    assert status == 'OK'
    status, state = ttt.play('X', 0)
    assert state == [ 'X', '_', '_', '_', '_', 'O', '_', '_', '_', ]
    assert status == 'TAKEN'
    status, state = ttt.play('O', 6)
    assert state == [ 'X', '_', '_', '_', '_', 'O', '_', '_', '_', ]
    assert status == 'BAD TURN'
    status, state = ttt.play('X', 1)
    assert state == [ 'X', 'X', '_', '_', '_', 'O', '_', '_', '_', ]
    assert status == 'OK'
    status, state = ttt.play('O', 6)
    assert state == [ 'X', 'X', '_', '_', '_', 'O', 'O', '_', '_', ]
    assert status == 'OK'
    status, state = ttt.play('X', 2)
    assert state == [ 'X', 'X', 'X', '_', '_', 'O', 'O', '_', '_', ]
    assert status == 'X WINS'
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.db import transaction
from django.db.models import get_model
from django.http import HttpResponseRedirect
from django.views.generic import (
    CreateView,
    DeleteView,
    UpdateView,
    TemplateView,
)

from braces.views import (
    LoginRequiredMixin,
    StaffuserRequiredMixin,
)

from base.view_utils import BaseMixin

from .models import (
    BlockError,
    Page,
    PageSection,
    Section,
)

def _get_block_model(page_section):
    block_model = get_model(
        page_section.section.block_app,
        page_section.section.block_model,
    )
    if not block_model:
        raise BlockError("Block model '{}.{}' does not exist.".format(
            page_section.section.block_app,
            page_section.section.block_model,
        ))
    return block_model

class ContentPageMixin(BaseMixin):
    """Page information."""

    def get_context_data(self, **kwargs):
        context = super(ContentPageMixin, self).get_context_data(**kwargs)
        context.update(dict(
            page=self.get_page(),
            pages=Page.objects.menu(),
        ))
        return context

    def get_page(self):
        menu = self.kwargs.get('menu', None)
        page = self.kwargs.get('page', None)
        if not page:
            raise BlockError("no 'page' parameter in url")
        try:
            if menu:
                return Page.objects.get(slug=page, slug_menu=menu)
            else:
                return Page.objects.get(slug=page)
        except Page.DoesNotExist:
            raise BlockError(
                "Page '{}', menu '{}' does not exist".format(page, menu)
            )

    def get_page_section(self):
        page = self.get_page()
        section = self.get_section()
        try:
            return PageSection.objects.get(page=page, section=section)
        except PageSection.DoesNotExist:
            raise BlockError(
                "Page section '{}/{}' does not exist".format(
                    page.slug,
                    section.slug,
                )
            )

    def get_section(self):
        section = self.kwargs.get('section', None)
        if not section:
            raise BlockError("no 'section' parameter in url")
        try:
            return Section.objects.get(slug=section)
        except Section.DoesNotExist:
            raise BlockError("Section '{}' does not exist".format(section))

    def get_success_url(self):
        return self.object.block.page_section.page.get_absolute_url()

class ContentCreateView(ContentPageMixin, BaseMixin, CreateView):

    def form_valid(self, form):
        self.object = form.save(commit=False)
        try:
            block = self.block_class(page_section=self.get_page_section())
        except AttributeError:
            raise BlockError(
                "You need to add the 'block_class' attribute "
                "to your '{}' class.".format(self.__class__.__name__)
            )
        block.save()
        self.object.block = block
        self.object.order = self.model.objects.next_order()
        return super(ContentCreateView, self).form_valid(form)

class ContentPublishView(BaseMixin, UpdateView):

    def form_valid(self, form):
        """Publish 'pending' content."""
        self.object = form.save(commit=False)
        self.object.block.publish(self.request.user)
        messages.info(
            self.request,
            "Published block, id {}".format(
                self.object.block.pk,
            )
        )
        return HttpResponseRedirect(self.get_success_url())

    def get_success_url(self):
        return self.object.block.page_section.page.get_absolute_url()

class ContentRemoveView(BaseMixin, UpdateView):

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.block.remove(self.request.user)
        messages.info(
            self.request,
            "Removed block {}".format(
                self.object.block.pk,
            )
        )
        return HttpResponseRedirect(self.get_success_url())

    def get_success_url(self):
        return self.object.block.page_section.page.get_absolute_url()

class ContentUpdateView(BaseMixin, UpdateView):

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.set_pending_edit()
        return super(ContentUpdateView, self).form_valid(form)

    def get_context_data(self, **kwargs):
        context = super(ContentUpdateView, self).get_context_data(**kwargs)
        context.update(dict(
            is_update=True,
        ))
        return context

    def get_section(self):
        return self.object.block.section

    def get_success_url(self):
        return self.object.block.page_section.page.get_absolute_url()

class ElementCreateView(BaseMixin, CreateView):
    """Update the parent (content) object so it knows it has been changed."""

    def form_valid(self, form):
        self.object = form.save(commit=False)
        content = self.object.get_parent()
        content.set_pending_edit()
        content.save()
        return super(ElementCreateView, self).form_valid(form)

class ElementDeleteView(BaseMixin, DeleteView):
    """Update the parent (content) object so it knows it has been changed."""

    def delete(self, request, *args, **kwargs):
        with transaction.atomic():
            result = super(ElementDeleteView, self).delete(
                request, *args, **kwargs
            )
            content = self.object.get_parent()
            content.set_pending_edit()
            content.save()
        return result

class ElementUpdateView(BaseMixin, UpdateView):
    """Update the parent (content) object so it knows it has been changed."""

    def form_valid(self, form):
        self.object = form.save(commit=False)
        content = self.object.get_parent()
        content.set_pending_edit()
        content.save()
        return super(ElementUpdateView, self).form_valid(form)

class PageTemplateMixin(object):

    def get_template_names(self) :
        page = self.get_page()
        if page.template_name:
            template_name = page.template_name
        else:
            raise BlockError(
                "Page '{}' has no 'template_name'".format(page.slug)
            )
        return [template_name,]

class PageDesignMixin(object):

    def get_context_data(self, **kwargs):
        context = super(PageDesignMixin, self).get_context_data(**kwargs)
        page = self.get_page()
        context.update(dict(design=True))
        for e in PageSection.objects.filter(page=page) :
            block_create_url = '{}_create_url'.format(e.section.slug)
            block_list_name = '{}_list'.format(e.section.slug)
            block_model = _get_block_model(e)
            kwargs = dict(section=e.section.slug)
            kwargs.update(page.get_url_kwargs())
            context.update({
                block_list_name: block_model.objects.pending(e),
            })
            if e.section.create_url_name:
                context.update({
                    block_create_url: reverse(
                        e.section.create_url_name, kwargs=kwargs
                    ),
                })
        return context

class PageDesignView(
        LoginRequiredMixin, StaffuserRequiredMixin,
        PageDesignMixin, PageTemplateMixin, ContentPageMixin, TemplateView):

    pass

class PageMixin(object):

    def get_context_data(self, **kwargs):
        context = super(PageMixin, self).get_context_data(**kwargs)
        page = self.get_page()
        context.update(dict(design=False))
        for e in PageSection.objects.filter(page=page):
            block_list_name = '{}_list'.format(e.section.slug)
            block_model = _get_block_model(e)
            context.update({
                block_list_name: block_model.objects.published(e),
            })
        return context

class PageView(
        PageMixin, PageTemplateMixin, ContentPageMixin, TemplateView):

    pass
<commit_msg>after page edit, we should return to design view<commit_after># -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.db import transaction
from django.db.models import get_model
from django.http import HttpResponseRedirect
from django.views.generic import (
    CreateView,
    DeleteView,
    UpdateView,
    TemplateView,
)

from braces.views import (
    LoginRequiredMixin,
    StaffuserRequiredMixin,
)

from base.view_utils import BaseMixin

from .models import (
    BlockError,
    Page,
    PageSection,
    Section,
)

def _get_block_model(page_section):
    block_model = get_model(
        page_section.section.block_app,
        page_section.section.block_model,
    )
    if not block_model:
        raise BlockError("Block model '{}.{}' does not exist.".format(
            page_section.section.block_app,
            page_section.section.block_model,
        ))
    return block_model

class ContentPageMixin(BaseMixin):
    """Page information."""

    def get_context_data(self, **kwargs):
        context = super(ContentPageMixin, self).get_context_data(**kwargs)
        context.update(dict(
            page=self.get_page(),
            pages=Page.objects.menu(),
        ))
        return context

    def get_page(self):
        menu = self.kwargs.get('menu', None)
        page = self.kwargs.get('page', None)
        if not page:
            raise BlockError("no 'page' parameter in url")
        try:
            if menu:
                return Page.objects.get(slug=page, slug_menu=menu)
            else:
                return Page.objects.get(slug=page)
        except Page.DoesNotExist:
            raise BlockError(
                "Page '{}', menu '{}' does not exist".format(page, menu)
            )

    def get_page_section(self):
        page = self.get_page()
        section = self.get_section()
        try:
            return PageSection.objects.get(page=page, section=section)
        except PageSection.DoesNotExist:
            raise BlockError(
                "Page section '{}/{}' does not exist".format(
                    page.slug,
                    section.slug,
                )
            )

    def get_section(self):
        section = self.kwargs.get('section', None)
        if not section:
            raise BlockError("no 'section' parameter in url")
        try:
            return Section.objects.get(slug=section)
        except Section.DoesNotExist:
            raise BlockError("Section '{}' does not exist".format(section))

    def get_success_url(self):
        return self.object.block.page_section.page.get_absolute_url()

class ContentCreateView(ContentPageMixin, BaseMixin, CreateView):

    def form_valid(self, form):
        self.object = form.save(commit=False)
        try:
            block = self.block_class(page_section=self.get_page_section())
        except AttributeError:
            raise BlockError(
                "You need to add the 'block_class' attribute "
                "to your '{}' class.".format(self.__class__.__name__)
            )
        block.save()
        self.object.block = block
        self.object.order = self.model.objects.next_order()
        return super(ContentCreateView, self).form_valid(form)

class ContentPublishView(BaseMixin, UpdateView):

    def form_valid(self, form):
        """Publish 'pending' content."""
        self.object = form.save(commit=False)
        self.object.block.publish(self.request.user)
        messages.info(
            self.request,
            "Published block, id {}".format(
                self.object.block.pk,
            )
        )
        return HttpResponseRedirect(self.get_success_url())

    def get_success_url(self):
        return self.object.block.page_section.page.get_design_url()

class ContentRemoveView(BaseMixin, UpdateView):

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.block.remove(self.request.user)
        messages.info(
            self.request,
            "Removed block {}".format(
                self.object.block.pk,
            )
        )
        return HttpResponseRedirect(self.get_success_url())

    def get_success_url(self):
        return self.object.block.page_section.page.get_design_url()

class ContentUpdateView(BaseMixin, UpdateView):

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.set_pending_edit()
        return super(ContentUpdateView, self).form_valid(form)

    def get_context_data(self, **kwargs):
        context = super(ContentUpdateView, self).get_context_data(**kwargs)
        context.update(dict(
            is_update=True,
        ))
        return context

    def get_section(self):
        return self.object.block.section

    def get_success_url(self):
        return self.object.block.page_section.page.get_design_url()

class ElementCreateView(BaseMixin, CreateView):
    """Update the parent (content) object so it knows it has been changed."""

    def form_valid(self, form):
        self.object = form.save(commit=False)
        content = self.object.get_parent()
        content.set_pending_edit()
        content.save()
        return super(ElementCreateView, self).form_valid(form)

class ElementDeleteView(BaseMixin, DeleteView):
    """Update the parent (content) object so it knows it has been changed."""

    def delete(self, request, *args, **kwargs):
        with transaction.atomic():
            result = super(ElementDeleteView, self).delete(
                request, *args, **kwargs
            )
            content = self.object.get_parent()
            content.set_pending_edit()
            content.save()
        return result

class ElementUpdateView(BaseMixin, UpdateView):
    """Update the parent (content) object so it knows it has been changed."""

    def form_valid(self, form):
        self.object = form.save(commit=False)
        content = self.object.get_parent()
        content.set_pending_edit()
        content.save()
        return super(ElementUpdateView, self).form_valid(form)

class PageTemplateMixin(object):

    def get_template_names(self) :
        page = self.get_page()
        if page.template_name:
            template_name = page.template_name
        else:
            raise BlockError(
                "Page '{}' has no 'template_name'".format(page.slug)
            )
        return [template_name,]

class PageDesignMixin(object):

    def get_context_data(self, **kwargs):
        context = super(PageDesignMixin, self).get_context_data(**kwargs)
        page = self.get_page()
        context.update(dict(design=True))
        for e in PageSection.objects.filter(page=page) :
            block_create_url = '{}_create_url'.format(e.section.slug)
            block_list_name = '{}_list'.format(e.section.slug)
            block_model = _get_block_model(e)
            kwargs = dict(section=e.section.slug)
            kwargs.update(page.get_url_kwargs())
            context.update({
                block_list_name: block_model.objects.pending(e),
            })
            if e.section.create_url_name:
                context.update({
                    block_create_url: reverse(
                        e.section.create_url_name, kwargs=kwargs
                    ),
                })
        return context

class PageDesignView(
        LoginRequiredMixin, StaffuserRequiredMixin,
        PageDesignMixin, PageTemplateMixin, ContentPageMixin, TemplateView):

    pass

class PageMixin(object):

    def get_context_data(self, **kwargs):
        context = super(PageMixin, self).get_context_data(**kwargs)
        page = self.get_page()
        context.update(dict(design=False))
        for e in PageSection.objects.filter(page=page):
            block_list_name = '{}_list'.format(e.section.slug)
            block_model = _get_block_model(e)
            context.update({
                block_list_name: block_model.objects.published(e),
            })
        return context

class PageView(
        PageMixin, PageTemplateMixin, ContentPageMixin, TemplateView):

    pass
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf -*-

# orm
from sqlalchemy import Column, ForeignKey, Integer, Float, String, Boolean
from sqlalchemy.sql.expression import desc, and_

# db
from db.database import db_session
from db.database import Base

# models
from models.accounts import Accounts
from models.totals import Totals
from models.users import UsersTable

# utils
from utils import *

class Expenses(object):

    object = None

    def __init__(self, user_id, user_type=None):
        # determine user type
        if not user_type:
            user_type = 'private' if UsersTable.query\
            .filter(and_(UsersTable.id == user_id, UsersTable.is_private == True)).first() else 'normal'
        if user_type == 'normal':
            self.object = NormalUserExpenses(user_id)
        else:
            self.object = PrivateUserExpenses(user_id)

    def __getattr__(self, name):
        print "calling: ", name, ' on ', self.object
        return getattr(self.object, name)

class ExpensesBase():

    user_id = None

    entries = None
    categories = None

    entry = None # cache

    totals = None

    def __init__(self, user_id):
        self.user_id = user_id

        self.totals = Totals(user_id)

    def get_entries(self, category_id=None, category_name=None, date_from=None, date_to=None, limit=None):
        if not self.entries:
            self.entries = ExpensesTable.query\
            .filter(ExpensesTable.user == self.user_id)\
            .join(ExpenseCategoriesTable)\
            .add_columns(ExpenseCategoriesTable.name, ExpenseCategoriesTable.slug)\
            .order_by(desc(ExpensesTable.date)).order_by(desc(ExpensesTable.id))\
            .outerjoin(ExpensesToLoansTable)\
            .outerjoin((UsersTable, (UsersTable.id == ExpensesToLoansTable.shared_with)))\
            .add_columns(UsersTable.name, UsersTable.slug)

        # provided category id
        if category_id:
            self.entries = self.entries.filter(ExpensesTable.category == category_id)

        # provided category name
        if category_name:
            self.entries = self.entries.filter(ExpenseCategoriesTable.name == category_name)

        # provided date range
        if date_from and date_to:
            self.entries = self.entries.filter(ExpensesTable.date >= date_from).filter(ExpensesTable.date <= date_to)

        # provided count
        if limit:
            self.entries = self.entries.limit(5)

        return self.entries

    def get_categories(self):
        if not self.categories:
            self.categories = ExpenseCategoriesTable.query\
            .filter(ExpenseCategoriesTable.user == self.user_id).order_by(ExpenseCategoriesTable.name)
        return self.categories

    def is_category(self, slug=None, name=None, id=None):
        categories = self.get_categories()
        for cat in categories:
            if slug:
                if cat.slug == slug:
                    return cat.id
                    break
            elif name:
                if cat.name == name:
                    return cat.id
                    break
            elif id:
                if cat.id == int(id):
                    return cat.id
                    break

    def add_category(self, name):
        c = ExpenseCategoriesTable(self.user_id, name)
        db_session.add(c)
        db_session.commit()

    def add_expense(self, date, description, amount, category_id=None, account_id=None, pass_thru=False):
        # add into uncategorized expenses if category not provided
        if not category_id:
            category_id = self.is_category(name="Uncategorized")
            if not category_id:
                # crete a new one
                c = ExpenseCategoriesTable(self.user_id, u'Uncategorized')
                db_session.add(c)
                db_session.commit()
                category_id = c.id

        # find default account if not provided
        if not account_id:
            acc = Accounts(self.user_id)
            account_id = acc.get_default_account()
            if not account_id:
                account_id = acc.add_default_account()

        # add the actual expense
        e = ExpensesTable(self.user_id, date, category_id, description, account_id, amount, pass_thru)
        db_session.add(e)
        db_session.commit()

        # update the totals
        self.totals.update_expense(amount, date)

        return e.id

    def edit_expense(self, date, amount, account_id, expense_id, description=None, category_id=None, pass_thru=False):
        e = self.get_simple_expense(expense_id)
        if e:
            # update the totals
            self.totals.update_expense(-float(e.amount), e.date)

            # category and description not provided?
            if not description: description = e.description
            if not category_id: category_id = e.category

            e.date, e.category, e.description, e.deduct_from, e.amount, e.pass_thru\
            = date, category_id, description, account_id, amount, pass_thru
            db_session.add(e)
            db_session.commit()

            # update the totals
            self.totals.update_expense(amount, date)

            return e # return so we see the updated values

    def edit_pass_thru_expense(self, description, category_id, expense_id):
        e = self.get_simple_expense(expense_id)
        if e:
            e.description, e.category = description, category_id
            db_session.add(e)
            db_session.commit()

    def get_simple_expense(self, expense_id):
        return ExpensesTable.query\
        .filter(and_(ExpensesTable.user == self.user_id, ExpensesTable.id == expense_id)).first()

    def get_expense(self, expense_id=None, loan_id=None):
        if loan_id:
            # fetch the expense through a loan
            self.entry = ExpensesTable.query\
            .outerjoin(ExpensesToLoansTable)\
            .filter(and_(ExpensesTable.user == self.user_id, ExpensesToLoansTable.loan == loan_id))\
            .add_columns(ExpensesToLoansTable.loan, ExpensesToLoansTable.shared_with, ExpensesToLoansTable.percentage,
                         ExpensesToLoansTable.original_amount)\
            .first()
        else:
            # if there is no cache or cache id does not match
            if not self.entry or self.entry[0].id != expense_id:
                self.entry = ExpensesTable.query\
                .filter(and_(ExpensesTable.user == self.user_id, ExpensesTable.id == expense_id))\
                .outerjoin(ExpensesToLoansTable).add_columns(ExpensesToLoansTable.loan, ExpensesToLoansTable.shared_with,
                                                             ExpensesToLoansTable.percentage,
                                                             ExpensesToLoansTable.original_amount)\
                .first()

        return self.entry

    def delete_expense(self, expense_id):
        e = self.get_expense(expense_id=expense_id)
        # update the totals
        self.totals.update_expense(-float(e[0].amount), e[0].date)

        # delete
        ExpensesTable.query.filter(ExpensesTable.id == expense_id).delete()
        db_session.commit()

    # blindly obey
    def link_to_loan(self, expense_id, loan_id, shared_with, percentage, original_amount):
        l = ExpensesToLoansTable(expense=expense_id, loan=loan_id, shared_with=shared_with, percentage=percentage,
                                 original_amount=original_amount)
        db_session.add(l)
        db_session.commit()

    # blindly obey and delete records for all entries associated with this loan
    def unlink_loan(self, loan_id):
        ExpensesToLoansTable.query\
        .filter(ExpensesToLoansTable.loan == loan_id).delete()
        db_session.commit()

    # blindly obey and delete records for all entries associated with this loan
    def modify_loan_link(self, loan_id, percentage, original_amount):
        le = ExpensesToLoansTable.query.filter(ExpensesToLoansTable.loan == loan_id)
        if le:
            for i in le:
                i.percentage, i.original_amount = percentage, original_amount
                db_session.add(i)
            db_session.commit()

class NormalUserExpenses(ExpensesBase):
    pass

class PrivateUserExpenses(ExpensesBase):

    def add_expense(self, date, description, amount, category_id=None, account_id=None, pass_thru=False):
        return None

    def link_to_loan(self, expense_id, loan_id, shared_with, percentage, original_amount):
        return None

    def get_expense(self, expense_id=None, loan_id=None):
        # so we match the returned list in normal call...
        class Temp(): id=None
        return [Temp()]

    def edit_expense(self, date, amount, account_id, expense_id, description=None, category_id=None, pass_thru=False):
        return None

    def delete_expense(self, expense_id):
        return None

class ExpenseCategoriesTable(Base):
    """Expense category of a user"""

    __tablename__ = 'expense_categories'
    id = Column(Integer, primary_key=True)
    user = Column('user_id', Integer, ForeignKey('users.id'))
    name = Column(String(50))
    slug = Column(String(50))

    def __init__(self, user=None, name=None):
        self.user = user
        self.name = name
        self.slug = slugify(name)

class ExpensesTable(Base):
    """Listing of expenses of various users"""

    __tablename__ = 'expenses'
    id = Column(Integer, primary_key=True)
    user = Column('user_id', Integer, ForeignKey('users.id'))
    date = Column(Integer)
    category = Column('category_id', Integer, ForeignKey('expense_categories.id'))
    description = Column(String(50))
    deduct_from = Column('account_id', Integer, ForeignKey('accounts.id'))
    amount = Column(Float(precision=2))
    pass_thru = Column(Boolean)

    def __init__(self, user=None, date=None, category=None, description=None, deduct_from=None, amount=None, pass_thru=None):
        self.user = user
        self.date = date
        self.category = category
        self.description = description
        self.deduct_from = deduct_from
        self.amount = amount
        self.pass_thru = pass_thru

class ExpensesToLoansTable(Base):
    """Linking shared expenses together"""

    __tablename__ = 'expenses_to_loans'
    id = Column(Integer, primary_key=True)
    expense = Column('expense_id', Integer, ForeignKey('expenses.id'))
    loan = Column('loan_id', Integer, ForeignKey('loans.id'))
    shared_with = Column('shared_with_user_id', Integer, ForeignKey('users.id')) # shortcut
    percentage = Column(Float(precision=2))
    original_amount = Column(Float(precision=2))

    def __init__(self, expense=None, loan=None, shared_with=None, percentage=None, original_amount=None):
        self.expense = expense
        self.loan = loan
        self.shared_with = shared_with
        self.percentage = percentage
        self.original_amount = original_amount<commit_msg>remove hardcoded value<commit_after>#!/usr/bin/python
# -*- coding: utf -*-

# orm
from sqlalchemy import Column, ForeignKey, Integer, Float, String, Boolean
from sqlalchemy.sql.expression import desc, and_

# db
from db.database import db_session
from db.database import Base

# models
from models.accounts import Accounts
from models.totals import Totals
from models.users import UsersTable

# utils
from utils import *

class Expenses(object):

    object = None

    def __init__(self, user_id, user_type=None):
        # determine user type
        if not user_type:
            user_type = 'private' if UsersTable.query\
            .filter(and_(UsersTable.id == user_id, UsersTable.is_private == True)).first() else 'normal'
        if user_type == 'normal':
            self.object = NormalUserExpenses(user_id)
        else:
            self.object = PrivateUserExpenses(user_id)

    def __getattr__(self, name):
        print "calling: ", name, ' on ', self.object
        return getattr(self.object, name)

class ExpensesBase():

    user_id = None

    entries = None
    categories = None

    entry = None # cache

    totals = None

    def __init__(self, user_id):
        self.user_id = user_id

        self.totals = Totals(user_id)

    def get_entries(self, category_id=None, category_name=None, date_from=None, date_to=None, limit=None):
        if not self.entries:
            self.entries = ExpensesTable.query\
            .filter(ExpensesTable.user == self.user_id)\
            .join(ExpenseCategoriesTable)\
            .add_columns(ExpenseCategoriesTable.name, ExpenseCategoriesTable.slug)\
            .order_by(desc(ExpensesTable.date)).order_by(desc(ExpensesTable.id))\
            .outerjoin(ExpensesToLoansTable)\
            .outerjoin((UsersTable, (UsersTable.id == ExpensesToLoansTable.shared_with)))\
            .add_columns(UsersTable.name, UsersTable.slug)

        # provided category id
        if category_id:
            self.entries = self.entries.filter(ExpensesTable.category == category_id)

        # provided category name
        if category_name:
            self.entries = self.entries.filter(ExpenseCategoriesTable.name == category_name)

        # provided date range
        if date_from and date_to:
            self.entries = self.entries.filter(ExpensesTable.date >= date_from).filter(ExpensesTable.date <= date_to)

        # provided count
        if limit:
            self.entries = self.entries.limit(limit)

        return self.entries

    def get_categories(self):
        if not self.categories:
            self.categories = ExpenseCategoriesTable.query\
            .filter(ExpenseCategoriesTable.user == self.user_id).order_by(ExpenseCategoriesTable.name)
        return self.categories

    def is_category(self, slug=None, name=None, id=None):
        categories = self.get_categories()
        for cat in categories:
            if slug:
                if cat.slug == slug:
                    return cat.id
                    break
            elif name:
                if cat.name == name:
                    return cat.id
                    break
            elif id:
                if cat.id == int(id):
                    return cat.id
                    break

    def add_category(self, name):
        c = ExpenseCategoriesTable(self.user_id, name)
        db_session.add(c)
        db_session.commit()

    def add_expense(self, date, description, amount, category_id=None, account_id=None, pass_thru=False):
        # add into uncategorized expenses if category not provided
        if not category_id:
            category_id = self.is_category(name="Uncategorized")
            if not category_id:
                # crete a new one
                c = ExpenseCategoriesTable(self.user_id, u'Uncategorized')
                db_session.add(c)
                db_session.commit()
                category_id = c.id

        # find default account if not provided
        if not account_id:
            acc = Accounts(self.user_id)
            account_id = acc.get_default_account()
            if not account_id:
                account_id = acc.add_default_account()

        # add the actual expense
        e = ExpensesTable(self.user_id, date, category_id, description, account_id, amount, pass_thru)
        db_session.add(e)
        db_session.commit()

        # update the totals
        self.totals.update_expense(amount, date)

        return e.id

    def edit_expense(self, date, amount, account_id, expense_id, description=None, category_id=None, pass_thru=False):
        e = self.get_simple_expense(expense_id)
        if e:
            # update the totals
            self.totals.update_expense(-float(e.amount), e.date)

            # category and description not provided?
            if not description: description = e.description
            if not category_id: category_id = e.category

            e.date, e.category, e.description, e.deduct_from, e.amount, e.pass_thru\
            = date, category_id, description, account_id, amount, pass_thru
            db_session.add(e)
            db_session.commit()

            # update the totals
            self.totals.update_expense(amount, date)

            return e # return so we see the updated values

    def edit_pass_thru_expense(self, description, category_id, expense_id):
        e = self.get_simple_expense(expense_id)
        if e:
            e.description, e.category = description, category_id
            db_session.add(e)
            db_session.commit()

    def get_simple_expense(self, expense_id):
        return ExpensesTable.query\
        .filter(and_(ExpensesTable.user == self.user_id, ExpensesTable.id == expense_id)).first()

    def get_expense(self, expense_id=None, loan_id=None):
        if loan_id:
            # fetch the expense through a loan
            self.entry = ExpensesTable.query\
            .outerjoin(ExpensesToLoansTable)\
            .filter(and_(ExpensesTable.user == self.user_id, ExpensesToLoansTable.loan == loan_id))\
            .add_columns(ExpensesToLoansTable.loan, ExpensesToLoansTable.shared_with, ExpensesToLoansTable.percentage,
                         ExpensesToLoansTable.original_amount)\
            .first()
        else:
            # if there is no cache or cache id does not match
            if not self.entry or self.entry[0].id != expense_id:
                self.entry = ExpensesTable.query\
                .filter(and_(ExpensesTable.user == self.user_id, ExpensesTable.id == expense_id))\
                .outerjoin(ExpensesToLoansTable).add_columns(ExpensesToLoansTable.loan, ExpensesToLoansTable.shared_with,
                                                             ExpensesToLoansTable.percentage,
                                                             ExpensesToLoansTable.original_amount)\
                .first()

        return self.entry

    def delete_expense(self, expense_id):
        e = self.get_expense(expense_id=expense_id)
        # update the totals
        self.totals.update_expense(-float(e[0].amount), e[0].date)

        # delete
        ExpensesTable.query.filter(ExpensesTable.id == expense_id).delete()
        db_session.commit()

    # blindly obey
    def link_to_loan(self, expense_id, loan_id, shared_with, percentage, original_amount):
        l = ExpensesToLoansTable(expense=expense_id, loan=loan_id, shared_with=shared_with, percentage=percentage,
                                 original_amount=original_amount)
        db_session.add(l)
        db_session.commit()

    # blindly obey and delete records for all entries associated with this loan
    def unlink_loan(self, loan_id):
        ExpensesToLoansTable.query\
        .filter(ExpensesToLoansTable.loan == loan_id).delete()
        db_session.commit()

    # blindly obey and delete records for all entries associated with this loan
    def modify_loan_link(self, loan_id, percentage, original_amount):
        le = ExpensesToLoansTable.query.filter(ExpensesToLoansTable.loan == loan_id)
        if le:
            for i in le:
                i.percentage, i.original_amount = percentage, original_amount
                db_session.add(i)
            db_session.commit()

class NormalUserExpenses(ExpensesBase):
    pass

class PrivateUserExpenses(ExpensesBase):

    def add_expense(self, date, description, amount, category_id=None, account_id=None, pass_thru=False):
        return None

    def link_to_loan(self, expense_id, loan_id, shared_with, percentage, original_amount):
        return None

    def get_expense(self, expense_id=None, loan_id=None):
        # so we match the returned list in normal call...
        class Temp(): id=None
        return [Temp()]

    def edit_expense(self, date, amount, account_id, expense_id, description=None, category_id=None, pass_thru=False):
        return None

    def delete_expense(self, expense_id):
        return None

class ExpenseCategoriesTable(Base):
    """Expense category of a user"""

    __tablename__ = 'expense_categories'
    id = Column(Integer, primary_key=True)
    user = Column('user_id', Integer, ForeignKey('users.id'))
    name = Column(String(50))
    slug = Column(String(50))

    def __init__(self, user=None, name=None):
        self.user = user
        self.name = name
        self.slug = slugify(name)

class ExpensesTable(Base):
    """Listing of expenses of various users"""

    __tablename__ = 'expenses'
    id = Column(Integer, primary_key=True)
    user = Column('user_id', Integer, ForeignKey('users.id'))
    date = Column(Integer)
    category = Column('category_id', Integer, ForeignKey('expense_categories.id'))
    description = Column(String(50))
    deduct_from = Column('account_id', Integer, ForeignKey('accounts.id'))
    amount = Column(Float(precision=2))
    pass_thru = Column(Boolean)

    def __init__(self, user=None, date=None, category=None, description=None, deduct_from=None, amount=None, pass_thru=None):
        self.user = user
        self.date = date
        self.category = category
        self.description = description
        self.deduct_from = deduct_from
        self.amount = amount
        self.pass_thru = pass_thru

class ExpensesToLoansTable(Base):
    """Linking shared expenses together"""

    __tablename__ = 'expenses_to_loans'
    id = Column(Integer, primary_key=True)
    expense = Column('expense_id', Integer, ForeignKey('expenses.id'))
    loan = Column('loan_id', Integer, ForeignKey('loans.id'))
    shared_with = Column('shared_with_user_id', Integer, ForeignKey('users.id')) # shortcut
    percentage = Column(Float(precision=2))
    original_amount = Column(Float(precision=2))

    def __init__(self, expense=None, loan=None, shared_with=None, percentage=None, original_amount=None):
        self.expense = expense
        self.loan = loan
        self.shared_with = shared_with
        self.percentage = percentage
        self.original_amount = original_amount<|endoftext|>


<commit_before>
import os
import sys
import json
import branchMatrix
import numpy as np

def main():
    fn = sys.argv[1]
    fm = sys.argv[2]
    branches = sys.argv[3]

    print "Annotating %s with header from %s" % (os.path.abspath(fn), os.path.abspath(fm))

    fo = open(fn)
    data = json.load(fo)
    fo.close()

    fo = open(fm)
    colheaders = fo.next().rstrip().split("\t")[1:]
    rowheaders = []
    termcat = []
    for line in fo:
        vs = line.rstrip().split("\t")
        rowheaders.append(vs[0])
        if vs[0].startswith("N:CLIN:TermCategory:NB"):
            termcat = vs[1:]
    fo.close()

    #print "%s data points %s names%s colheaders %s rowheaders max %s" % (len(data["f1"]), len(data["nByi"]), len(colheaders), len(rowheaders), max((int(n) for n in data["nByi"])))
    #print json.dumps(colheaders)
    #print json.dumps(data["nByi"])
    nByi = [colheaders[int(n)] for n in data["nByi"]]
    data["nByi"] = nByi

    fmiByHeader = dict((key, value) for (value, key) in enumerate(colheaders))
    rank = dict((int(key), int(value)) for (value, key) in enumerate(data["r1"]))
    sortedi = np.array([rank[key] for key in sorted(rank.keys())])
    fmis = np.array([fmiByHeader[nByi[i]] for i in sortedi])
    data["termcat"] = [int(n) for n in list(np.array(termcat)[fmis])]
    for key in ["f1","f2","r1","r2","d","nByi"]:
        data[key] = list(np.array(data[key])[sortedi])

    data["iByn"] = dict((key, value) for (value, key) in enumerate(data["nByi"]))

    print "Overwriting %s with version annotated for branch vis" % (fn)
    fo = open(fn, "w")
    json.dump(data, fo, indent=2)
    fo.close()

    fo = open(fn)
    data = json.load(fo)
    fo.close()

    data["branches"] = branchMatrix.branchMatrix(fn, fm, branches)

    print "Overwriting %s with version including branch data" % (fn)
    fo = open(fn, "w")
    json.dump(data, fo, indent=2)
    fo.close()

if __name__ == '__main__':
    main()<commit_msg>fixed int parseing<commit_after>
import os
import sys
import json
import branchMatrix
import numpy as np

def main():
    fn = sys.argv[1]
    fm = sys.argv[2]
    branches = sys.argv[3]

    print "Annotating %s with header from %s" % (os.path.abspath(fn), os.path.abspath(fm))

    fo = open(fn)
    data = json.load(fo)
    fo.close()

    fo = open(fm)
    colheaders = fo.next().rstrip().split("\t")[1:]
    rowheaders = []
    termcat = []
    for line in fo:
        vs = line.rstrip().split("\t")
        rowheaders.append(vs[0])
        if vs[0].startswith("N:CLIN:TermCategory:NB"):
            termcat = vs[1:]
    fo.close()

    #print "%s data points %s names%s colheaders %s rowheaders max %s" % (len(data["f1"]), len(data["nByi"]), len(colheaders), len(rowheaders), max((int(n) for n in data["nByi"])))
    #print json.dumps(colheaders)
    #print json.dumps(data["nByi"])
    nByi = [colheaders[int(n)] for n in data["nByi"]]
    data["nByi"] = nByi

    fmiByHeader = dict((key, value) for (value, key) in enumerate(colheaders))
    rank = dict((int(key), int(value)) for (value, key) in enumerate(data["r1"]))
    sortedi = np.array([rank[key] for key in sorted(rank.keys())])
    fmis = np.array([fmiByHeader[nByi[i]] for i in sortedi])
    data["termcat"] = [int(float(n)) for n in list(np.array(termcat)[fmis])]
    for key in ["f1","f2","r1","r2","d","nByi"]:
        data[key] = list(np.array(data[key])[sortedi])

    data["iByn"] = dict((key, value) for (value, key) in enumerate(data["nByi"]))

    print "Overwriting %s with version annotated for branch vis" % (fn)
    fo = open(fn, "w")
    json.dump(data, fo, indent=2)
    fo.close()

    fo = open(fn)
    data = json.load(fo)
    fo.close()

    data["branches"] = branchMatrix.branchMatrix(fn, fm, branches)

    print "Overwriting %s with version including branch data" % (fn)
    fo = open(fn, "w")
    json.dump(data, fo, indent=2)
    fo.close()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Commands that update or process the application data.
"""
import app_config
import dataset
import logging
import requests

from dateutil import parser
from fabric.api import local, task
from facebook import GraphAPI
from scrapers.utils import get_art_root_url

SECRETS = app_config.get_secrets()
FACEBOOK_USER = 'NPR'

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

@task
def local_bootstrap():
    local('dropdb --if-exists %s' % app_config.PROJECT_SLUG)
    local('createdb %s' % app_config.PROJECT_SLUG)

@task
def drop_tables():
    db = dataset.connect(app_config.POSTGRES_URL)

    for table_name in db.tables:
        print 'Dropping %s' % table_name
        table = db[table_name]
        table.drop()

@task
def fix_facebook_ids():
    """
    Retroactively adds Facebook IDs to Facebook table
    """
    db = dataset.connect(app_config.POSTGRES_URL)
    table = db['facebook']

    min_created_time_result = list(db.query('select min(created_time) as min_created_time from facebook'))
    min_created_time = min_created_time_result[0]['min_created_time']

    graph = GraphAPI(SECRETS['FACEBOOK_TOKEN'])
    profile = graph.get_object(FACEBOOK_USER)
    posts = graph.get_connections(profile['id'], 'posts')

    done = False
    while not done:
        print 'processing batch of 25'
        for post in posts['data']:
            created_time = parser.parse(post['created_time'], ignoretz=True)
            if created_time < min_created_time:
                done = True
                break

            if 'link' not in post.keys():
                print 'skipping %s (no link)' % post['id']
                continue

            link = post['link'].split('?')[0]
            print 'updating %s (%s)' % (link, post['id'])
            table.update({
                'facebook_id': post['id'],
                'link_url': link,
            }, ['link_url'])

        posts = requests.get(posts['paging']['next']).json()

@task
def fix_fb_art_urls():
    db = dataset.connect(app_config.POSTGRES_URL)

    fb = db['facebook']
    for row in fb.all():
        if row['art_url']:
            update = {
                'art_root_url': get_art_root_url(row['art_url']),
                'id': row['id']
            }
            print 'updating %s' % update
            fb.update(update, ['id'])

@task
def fix_homepage_art_urls():
    db = dataset.connect(app_config.POSTGRES_URL)
    hp = db['homepage']
    for row in hp.all():
        if row['homepage_art_url']:
            update = {
                'homepage_art_root_url': get_art_root_url(row['homepage_art_url']),
                'id': row['id']
            }
            print 'updating %s' % update
            hp.update(update, ['id'])

@task
def fix_seamus_art_urls():
    db = dataset.connect(app_config.POSTGRES_URL)
    seamus = db['seamus']
    for row in seamus.all():
        if row['lead_art_url']:
            update = {
                'lead_art_root_url': get_art_root_url(row['lead_art_url']),
                'id': row['id']
            }
            print 'updating %s' % update
            seamus.update(update, ['id'])

@task
def dump_db(directory):
    """
    Dump the database. Directory required.
    """
    db = dataset.connect(app_config.POSTGRES_URL)
    local('mkdir -p {0}'.format(directory))
    for table_name in db.tables:
        table = db[table_name]
        results = table.all()
        backup_filename = '{0}/{1}.csv'.format(directory, table_name)
        with open(backup_filename, 'w') as f:
            dataset.freeze(results, format='csv', fileobj=f)
<commit_msg>add db shell fab command<commit_after>#!/usr/bin/env python

"""
Commands that update or process the application data.
"""
import app_config
import dataset
import ipdb
import logging
import requests

from dateutil import parser
from fabric.api import local, task
from facebook import GraphAPI
from scrapers.utils import get_art_root_url

SECRETS = app_config.get_secrets()
FACEBOOK_USER = 'NPR'

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

@task
def local_bootstrap():
    local('dropdb --if-exists %s' % app_config.PROJECT_SLUG)
    local('createdb %s' % app_config.PROJECT_SLUG)

@task
def drop_tables():
    db = dataset.connect(app_config.POSTGRES_URL)

    for table_name in db.tables:
        print 'Dropping %s' % table_name
        table = db[table_name]
        table.drop()

@task
def fix_facebook_ids():
    """
    Retroactively adds Facebook IDs to Facebook table
    """
    db = dataset.connect(app_config.POSTGRES_URL)
    table = db['facebook']

    min_created_time_result = list(db.query('select min(created_time) as min_created_time from facebook'))
    min_created_time = min_created_time_result[0]['min_created_time']

    graph = GraphAPI(SECRETS['FACEBOOK_TOKEN'])
    profile = graph.get_object(FACEBOOK_USER)
    posts = graph.get_connections(profile['id'], 'posts')

    done = False
    while not done:
        print 'processing batch of 25'
        for post in posts['data']:
            created_time = parser.parse(post['created_time'], ignoretz=True)
            if created_time < min_created_time:
                done = True
                break

            if 'link' not in post.keys():
                print 'skipping %s (no link)' % post['id']
                continue

            link = post['link'].split('?')[0]
            print 'updating %s (%s)' % (link, post['id'])
            table.update({
                'facebook_id': post['id'],
                'link_url': link,
            }, ['link_url'])

        posts = requests.get(posts['paging']['next']).json()

@task
def fix_fb_art_urls():
    db = dataset.connect(app_config.POSTGRES_URL)

    fb = db['facebook']
    for row in fb.all():
        if row['art_url']:
            update = {
                'art_root_url': get_art_root_url(row['art_url']),
                'id': row['id']
            }
            print 'updating %s' % update
            fb.update(update, ['id'])

@task
def fix_homepage_art_urls():
    db = dataset.connect(app_config.POSTGRES_URL)
    hp = db['homepage']
    for row in hp.all():
        if row['homepage_art_url']:
            update = {
                'homepage_art_root_url': get_art_root_url(row['homepage_art_url']),
                'id': row['id']
            }
            print 'updating %s' % update
            hp.update(update, ['id'])

@task
def fix_seamus_art_urls():
    db = dataset.connect(app_config.POSTGRES_URL)
    seamus = db['seamus']
    for row in seamus.all():
        if row['lead_art_url']:
            update = {
                'lead_art_root_url': get_art_root_url(row['lead_art_url']),
                'id': row['id']
            }
            print 'updating %s' % update
            seamus.update(update, ['id'])

@task
def db_shell():
    db = dataset.connect(app_config.POSTGRES_URL)
    print 'Dropping to interactive shell. The `db` variable has access to the current live database.'
    ipdb.set_trace()
    pass

@task
def dump_db(directory):
    """
    Dump the database. Directory required.
    """
    db = dataset.connect(app_config.POSTGRES_URL)
    local('mkdir -p {0}'.format(directory))
    for table_name in db.tables:
        table = db[table_name]
        results = table.all()
        backup_filename = '{0}/{1}.csv'.format(directory, table_name)
        with open(backup_filename, 'w') as f:
            dataset.freeze(results, format='csv', fileobj=f)
<|endoftext|>


<commit_before>"""
Copyright 2016 Brocade Communications Systems, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from st2actions.runners.pythonrunner import Action

class ExcelAction(Action):
    def __init__(self, config):
        super(ExcelAction, self).__init__(config)
        self._excel_file = self.config['excel_file']
        self._key_column = self.config['key_column']
        self._variable_name_row = self.config['variable_name_row']
        self._lock_retries = self.config['lock_file_retries']
        self._lock_delay = self.config['lock_file_delay']
<commit_msg>Delete ExcelAction.py<commit_after><|endoftext|>