<commit_before>import math
import random

import numpy as np
from matplotlib import pyplot as plt

import seaborn as sns

def generate(intersect, slope, sensor_sigma, occupant_range, occupant_sigma, datapoints=1000):
    """Generates fake sensor data"""

    fake_data = {
        'occupants': [],
        'reading': []}

    for _ in range(datapoints):
        real_occupants = random.randrange(occupant_range + 1)
        noisy_occupants = max(0, random.gauss(real_occupants, occupant_sigma))
        reading = intersect + noisy_occupants * slope
        reading = random.gauss(reading, sensor_sigma)
        fake_data['occupants'].append(real_occupants)
        fake_data['reading'].append(reading)

    return fake_data

def plot(data, filename):
    plt.cla()
    plt.scatter(x, y)
    slope, intercept = np.polyfit(data["occupants"], data["reading"], 1)
    occupant_range = np.array(
        range(min(data["occupants"]), max(data["occupants"]))
    )
    fit = occupant_range * slope + intercept
    plt.plot(occupant_range, fit)

def fit(data):

    slope, intercept = np.polyfit(data["occupants"], data["reading"], 1)
    error = 0.0
    n_samples = len(data["occupants"])
    for occupants, reading in zip(data["occupants"], data["reading"]):
        error += (occupants * slope + intercept - reading)**2

    sigma = np.sqrt(error / (n_samples - 1))

    def sensor_model(occupants):
        return occupants * slope + intercept, sigma

    def predictor(sensor_reading):
        return (sensor_reading - intercept) / slope, sigma / slope

    return sensor_model, predictor

def round_up(number, scale):
    return int(math.ceil(number / float(scale)) * scale)

def round_down(number, scale):
    return int(math.floor(number / float(scale)) * scale)

def plot_sensor_model(data, sensor_model, filename, round_level=10):
    occupant_vector = np.array(
        range(min(data["occupants"]) - 1, max(data["occupants"]) + 2)
    )
    fit_vector = np.array([sensor_model(o)[0] for o in occupant_vector])
    _, sigma = sensor_model(occupant_vector[0])
    error_vector = [sigma] * len(occupant_vector)

    plt.clf()
    ax = plt.gca()
    ax.set_xlim([occupant_vector[0], occupant_vector[-1]])
    ax.set_ylim([round_down(min(data["reading"]), round_level),
                 round_up(max(data["reading"]), round_level)])

    ax = sns.regplot(
        x=np.array(data["occupants"]),
        y=np.array(data["reading"]),
        marker='x',
        fit_reg=False
    )
    ax.plot(occupant_vector, fit_vector)

    x_range = np.linspace(ax.get_xlim()[0], ax.get_xlim()[1], 100)
    y_range = np.linspace(ax.get_ylim()[0], ax.get_ylim()[1], 100)

    xx, yy = np.meshgrid(x_range, y_range)
    zz = xx + yy

    for i in range(len(x_range)):
        for j in range(len(y_range)):
            zz[j, i] = gaussian(yy[j, i], *sensor_model(xx[j, i]))

    pal = sns.light_palette("green", as_cmap=True)

    im = plt.imshow(zz,  interpolation='bilinear', origin='lower',
                    cmap=pal, alpha=0.5, aspect='auto',
                    extent=(x_range[0], x_range[-1], y_range[0], y_range[-1]))
    cb = plt.colorbar(im, orientation='vertical', ticks=[
        gaussian(3 * sigma, 0, sigma),
        gaussian(2 * sigma, 0, sigma),
        gaussian(sigma, 0, sigma),
        gaussian(0, 0, sigma),
    ],
        drawedges=True
    )
    cb.set_ticklabels(['$3 \sigma$', '$2 \sigma$',
                       '$\sigma$', 'max'], update_ticks=True)
    plt.savefig(filename)

def plot_predictor(reading_range, predictor, filename, round_level=1, readings=[]):

    palette = sns.color_palette()

    reading_vector = np.linspace(*reading_range)
    fit_vector = np.array([predictor(r)[0] for r in reading_vector])
    _, sigma = predictor(reading_vector[0])
    # error_vector = [sigma] * len(reading_vector)

    plt.clf()
    ax = plt.gca()
    ax.set_xlim([reading_vector[0], reading_vector[-1]])
    ax.set_ylim([round_down(fit_vector[0], round_level),
                 round_up(fit_vector[-1], round_level)])

    ax.plot(reading_vector, fit_vector, color=palette[1])

    x_range = np.linspace(ax.get_xlim()[0], ax.get_xlim()[1], 100)
    y_range = np.linspace(ax.get_ylim()[0], ax.get_ylim()[1], 100)

    xx, yy = np.meshgrid(x_range, y_range)
    zz = xx + yy

    for i in range(len(x_range)):
        for j in range(len(y_range)):
            zz[j, i] = gaussian(yy[j, i], *predictor(xx[j, i]))

    if readings:
        ax.vlines(readings, ax.get_ylim()[0], ax.get_ylim()[-1], linestyles='dotted')

    pal = sns.light_palette("green", as_cmap=True)

    im = plt.imshow(zz,  interpolation='bilinear', origin='lower',
                    cmap=pal, alpha=0.5, aspect='auto',
                    extent=(x_range[0], x_range[-1], y_range[0], y_range[-1]))
    cb = plt.colorbar(im, orientation='vertical', ticks=[
        gaussian(3 * sigma, 0, sigma),
        gaussian(2 * sigma, 0, sigma),
        gaussian(sigma, 0, sigma),
        gaussian(0, 0, sigma),
    ],
        drawedges=True
    )
    cb.set_ticklabels(['$3 \sigma$', '$2 \sigma$',
                       '$\sigma$', 'max'], update_ticks=True)
    plt.savefig(filename)

def plot_readings(reading_vector, predictor, filename, x_range=[0,15], fuse=False):
    plt.clf()
    ax = plt.gca()
    x_vector = np.linspace(*x_range)
    palette = sns.color_palette()
    fusion = None
    for reading in reading_vector:
        occupants, sigma = predictor(reading)
        if fuse:
            if not fusion:
                fusion = occupants, sigma
            else:
                fusion = bayesian_update(fusion, (occupants, sigma))
        y_vector = 100*gaussian(x_vector, occupants, sigma)
        ax.plot(x_vector, y_vector, color=palette[1])
        ax.vlines(occupants, 0, max(y_vector), linestyles='dotted', label='{}ppm'.format(reading))

    if fuse:
        y_vector = 100*gaussian(x_vector, *fusion)
        ax.plot(x_vector, y_vector, color=palette[3])

    ax.set_ylim(0, 20)
    ax.set_xlim(*x_range)
    plt.savefig(filename)

def gaussian(x, mu, sig):
    return np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) / (sig * np.sqrt(2. * np.pi))

def bayesian_update(gaussian_a, gaussian_b):
    mu_a, sigma_a = gaussian_a
    mu_b, sigma_b = gaussian_b
    mu = ((sigma_a**2)*mu_b + (sigma_b**2)*mu_a)/(sigma_a**2 + sigma_b**2)
    sigma = np.sqrt(((sigma_a*sigma_b)**2)/(sigma_a**2 + sigma_b**2))

    return mu, sigma

if __name__ == "__main__":

    co2_data = generate(350, 60, 10, 15, 5, 250)
    co2_sensor_model, co2_predictor = fit(co2_data)
    plot_sensor_model(co2_data, co2_sensor_model, 'co2_experiment.png', round_level=500)
    plot_predictor([0, 1500], co2_predictor, 'co2_predictor.png', round_level=10)
    plot_predictor([0, 1500], co2_predictor, 'co2_predictor1.png', round_level=10, readings=[733])
    plot_predictor([0, 1500], co2_predictor, 'co2_predictor2.png', round_level=10, readings=[733, 1037])
    plot_predictor([0, 1500], co2_predictor, 'co2_predictor3.png', round_level=10, readings=[733, 790, 1037, 500, 699])

    plot_readings([733], co2_predictor, 'co2_readings1.png', x_range=[0, 15])
    plot_readings([733, 1037], co2_predictor, 'co2_readings2a.png', x_range=[0, 15], fuse=False)
    plot_readings([733, 1037], co2_predictor, 'co2_readings2b.png', x_range=[0, 15], fuse=True)
    plot_readings([733, 790, 1037, 500, 699], co2_predictor, 'co2_readings3.png', x_range=[0, 15], fuse=True)

    temp_data = generate(19, 0.6, 0.5, 15, 5, 250)
    temp_sensor_model, temp_predictor = fit(temp_data)
    plot_sensor_model(temp_data, temp_sensor_model, 'temp_experiment.png', round_level=5)
    plot_predictor([10, 40], temp_predictor, 'temp_predictor.png', round_level=10)
<commit_msg>split things up in many classes<commit_after>import math
import random

import numpy as np
from matplotlib import pyplot as plt

import seaborn as sns

class Sensor(object):

    def __init__(self, name, **kwargs):
        self.name = name
        for key, value in kwargs.items():
            setattr(self, key, value)

    def read(self, variable):
        reading = self.intersect + variable * self.slope
        return random.gauss(reading, self.sigma)

    def fit(self, data):
        slope, intercept = np.polyfit(
            [o for o, r in data], [r for o, r in data], 1)
        error = 0.0
        n_samples = len(data)
        for occupants, reading in data:
            error += (occupants * slope + intercept - reading)**2

        sigma = np.sqrt(error / (n_samples - 1))

        def model(occupants):
            return occupants * slope + intercept, sigma

        def predictor(sensor_reading):
            return (sensor_reading - intercept) / slope, sigma / slope

        self.model = model
        self.predictor = predictor

class TrainCar(object):

    max_occupants = 120

    def __init__(self, occupants=0):
        self.sigma = self.max_occupants / 5
        self.occupants = occupants
        self.sensor_array = [
            Sensor("co2", intersect=350, slope=15, sigma=10),
            Sensor("temp", intersect=19, slope=0.6, sigma=0.5)
        ]

    def read_sensors(self):
        reading_dict = {}
        for sensor in self.sensor_array:
            occupants = max(0, random.gauss(self.occupants, self.sigma))
            reading_dict[sensor.name] = sensor.read(occupants)
        return reading_dict

    def run_experiment(self, datapoints=1000):
        """Generates fake sensor data"""

        data = []

        for _ in range(datapoints):
            self.occupants = random.randrange(self.max_occupants + 1)
            data.append((self.occupants, self.read_sensors()))

        for sensor in self.sensor_array:
            sensor.fit([(o, r[sensor.name]) for o, r in data])

        self.experiment_data = data

def round_up(number, scale):
    return int(math.ceil(number / float(scale)) * scale)

def round_down(number, scale):
    return int(math.floor(number / float(scale)) * scale)

def plot_sensor_model(data, sensor_model, filename, round_level=10):
    x_array = np.array([o for o, r in data])
    y_array = np.array([r for o, r in data])
    occupant_vector = np.array( range(min(x_array) - 1, max(x_array) + 2) )
    fit_vector = np.array([sensor_model(o)[0] for o in occupant_vector])
    _, sigma = sensor_model(occupant_vector[0])
    error_vector = [sigma] * len(occupant_vector)

    plt.clf()
    ax = plt.gca()
    ax.set_xlim([occupant_vector[0], occupant_vector[-1]])
    ax.set_ylim([round_down(min(y_array), round_level),
                 round_up(max(y_array), round_level)])

    ax = sns.regplot( x=x_array, y=y_array, marker='x', fit_reg=False )
    ax.plot(occupant_vector, fit_vector)

    x_range = np.linspace(ax.get_xlim()[0], ax.get_xlim()[1], 100)
    y_range = np.linspace(ax.get_ylim()[0], ax.get_ylim()[1], 100)

    xx, yy = np.meshgrid(x_range, y_range)
    zz = xx + yy

    for i in range(len(x_range)):
        for j in range(len(y_range)):
            zz[j, i] = gaussian(yy[j, i], *sensor_model(xx[j, i]))

    pal = sns.light_palette("green", as_cmap=True)

    im = plt.imshow(zz,  interpolation='bilinear', origin='lower',
                    cmap=pal, alpha=0.5, aspect='auto',
                    extent=(x_range[0], x_range[-1], y_range[0], y_range[-1]))
    cb = plt.colorbar(im, orientation='vertical', ticks=[
        gaussian(3 * sigma, 0, sigma),
        gaussian(2 * sigma, 0, sigma),
        gaussian(sigma, 0, sigma),
        gaussian(0, 0, sigma),
    ],
        drawedges=True
    )
    cb.set_ticklabels(['$3 \sigma$', '$2 \sigma$',
                       '$\sigma$', 'max'], update_ticks=True)
    plt.savefig(filename)

def plot_predictor(reading_range, predictor, filename, round_level=1, readings=[]):

    palette = sns.color_palette()

    reading_vector = np.linspace(*reading_range)
    fit_vector = np.array([predictor(r)[0] for r in reading_vector])
    _, sigma = predictor(reading_vector[0])
    # error_vector = [sigma] * len(reading_vector)

    plt.clf()
    ax = plt.gca()
    ax.set_xlim([reading_vector[0], reading_vector[-1]])
    ax.set_ylim([round_down(fit_vector[0], round_level),
                 round_up(fit_vector[-1], round_level)])

    ax.plot(reading_vector, fit_vector, color=palette[1])

    x_range = np.linspace(ax.get_xlim()[0], ax.get_xlim()[1], 100)
    y_range = np.linspace(ax.get_ylim()[0], ax.get_ylim()[1], 100)

    xx, yy = np.meshgrid(x_range, y_range)
    zz = xx + yy

    for i in range(len(x_range)):
        for j in range(len(y_range)):
            zz[j, i] = gaussian(yy[j, i], *predictor(xx[j, i]))

    if readings:
        ax.vlines(readings, ax.get_ylim()[
                  0], ax.get_ylim()[-1], linestyles='dotted')

    pal = sns.light_palette("green", as_cmap=True)

    im = plt.imshow(zz,  interpolation='bilinear', origin='lower',
                    cmap=pal, alpha=0.5, aspect='auto',
                    extent=(x_range[0], x_range[-1], y_range[0], y_range[-1]))
    cb = plt.colorbar(im, orientation='vertical', ticks=[
        gaussian(3 * sigma, 0, sigma),
        gaussian(2 * sigma, 0, sigma),
        gaussian(sigma, 0, sigma),
        gaussian(0, 0, sigma),
    ],
        drawedges=True
    )
    cb.set_ticklabels(['$3 \sigma$', '$2 \sigma$',
                       '$\sigma$', 'max'], update_ticks=True)
    plt.savefig(filename)

def plot_readings(reading_vector, predictor, filename, x_range=[0, 15], fuse=False):
    plt.clf()
    ax = plt.gca()
    x_vector = np.linspace(*x_range)
    palette = sns.color_palette()
    fusion = None
    for reading in reading_vector:
        occupants, sigma = predictor(reading)
        if fuse:
            if not fusion:
                fusion = occupants, sigma
            else:
                fusion = bayesian_update(fusion, (occupants, sigma))
        y_vector = 100 * gaussian(x_vector, occupants, sigma)
        ax.plot(x_vector, y_vector, color=palette[1])
        ax.vlines(occupants, 0, max(y_vector), linestyles='dotted',
                  label='{}ppm'.format(reading))

    if fuse:
        y_vector = 100 * gaussian(x_vector, *fusion)
        ax.plot(x_vector, y_vector, color=palette[3])

    ax.set_ylim(0, 10)
    ax.set_xlim(*x_range)
    plt.savefig(filename)

def gaussian(x, mu, sig):
    return np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) / (sig * np.sqrt(2. * np.pi))

def bayesian_update(gaussian_a, gaussian_b):
    mu_a, sigma_a = gaussian_a
    mu_b, sigma_b = gaussian_b
    mu = ((sigma_a**2) * mu_b + (sigma_b**2)
          * mu_a) / (sigma_a**2 + sigma_b**2)
    sigma = np.sqrt(((sigma_a * sigma_b)**2) / (sigma_a**2 + sigma_b**2))

    return mu, sigma

if __name__ == "__main__":

    train_car = TrainCar()
    co2_sensor = train_car.sensor_array[0]
    experiment_data = train_car.run_experiment(datapoints=250)
    co2_data = [(o, r["co2"]) for o, r in experiment_data]

    # co2_data = generate(350, 60, 10, 15, 5, 250)
    # co2_sensor_model, co2_predictor = fit(co2_data)
    plot_sensor_model(co2_data, co2_sensor.model,
                      'co2_experiment.png', round_level=500)
    plot_predictor([0, 2500], co2_sensor.predictor,
                   'co2_predictor.png', round_level=10)
    plot_predictor([0, 2500], co2_sensor.predictor,
                   'co2_predictor1.png', round_level=10, readings=[733])
    plot_predictor([0, 2500], co2_sensor.predictor, 'co2_predictor2.png',
                   round_level=10, readings=[733, 1037])
    plot_predictor([0, 2500], co2_sensor.predictor, 'co2_predictor3.png',
                   round_level=10, readings=[733, 790, 1037, 500, 699])

    plot_readings([733], co2_sensor.predictor, 'co2_readings1.png', x_range=[0, train_car.max_occupants])
    plot_readings([733, 1037], co2_sensor.predictor,
                  'co2_readings2a.png', x_range=[0, train_car.max_occupants], fuse=False)
    plot_readings([733, 1037], co2_sensor.predictor,
                  'co2_readings2b.png', x_range=[0, train_car.max_occupants], fuse=True)
    plot_readings([733, 790, 1037, 500, 699], co2_sensor.predictor,
                  'co2_readings3.png', x_range=[0, train_car.max_occupants], fuse=True)

    # temp_data = generate(19, 0.6, 0.5, 15, 5, 250)
    # temp_sensor_model, temp_predictor = fit(temp_data)
    # plot_sensor_model(temp_data, temp_sensor_model,
    #                   'temp_experiment.png', round_level=5)
    # plot_predictor([10, 40], temp_predictor,
    #                'temp_predictor.png', round_level=10)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import httplib2
import os
import json

from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
from termcolor import cprint

SCOPES = 'https://www.googleapis.com/auth/spreadsheets'
SECRET = 'records_auth.json'
APP_NAME = 'ward_members_auth'
SHEET_ID = '13zdxFnJLJ_7IfWQxOBm0sZPkIL5bHZ0RE29o_T27Hd0'
RANGE='New Members 2017'

FIRST = -1
LAST = -1
DOB  = -1
APT = -1
PHONE = -1
PULLED = -1
MRN = -1
BISHOP = -1
EMAIL = -1
CONTACTED = -1

LENGTH = -1

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()

except ImportError:
    flags = None

def get_credentials():

    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')

    if not os.path.exists(credential_dir):
        os.mkdirs(credential_dir)

    credential_path = os.path.join(credential_dir, 'sheets.googleapis.com-new-members.json')

    store = Storage(credential_path)
    credentials = store.get()

    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(SECRET, SCOPES)
        flow.user_agent = APP_NAME
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        print 'storing credentials to', credential_path

    return credentials

def build_service():

    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())

    discovery_url = ('https://sheets.googleapis.com/$discovery/rest?' 'version=v4')

    service = discovery.build('sheets', 'v4', http=http, discoveryServiceUrl=discovery_url)

    return service

#gets a dump of what's in Google Sheets
def get_data():

    service = build_service()

    result = service.spreadsheets().values().get(spreadsheetId = SHEET_ID, range = RANGE).execute()

    values = result.get('values', []) #what in the world does this do??

    if not values:
        raise ValueError('no data found')
    else:
        return values

#values is a list of lists
def set_columns(values):

    global LENGTH
    LENGTH = len(values[0])

    for i, cell in enumerate(values[0]):
        if cell == 'First Name':
            global FIRST
            FIRST = i
        elif cell == 'Last Name':
            global LAST
            LAST = i
        elif cell == 'Birth Month':
            global BIRTH_MONTH
            BIRTH_MONTH = i
        elif cell == 'Birth Day':
            global BIRTH_DAY
            BIRTH_DAY = i
        elif cell == 'Birth Year':
            global BIRTH_YEAR
            cell == i
        elif cell == 'Apartment':
            global APT
            APT = i
        elif cell == 'Phone Number':
            global PHONE
            PHONE = i
        elif cell == 'Email Address':
            global PERSONAL_EMAIL
            PERSONAL_EMAIL = i
        elif cell == 'Records Pulled':
            global PULLED
            PULLED = i
        elif cell == 'MRN':
            global MRN
            MRN = i
        elif cell == 'Former Bishop':
            global BISHOP
            BISHOP = i
        elif cell == 'Former Bishop Email':
            global EMAIL
            EMAIL = i
        else:
            continue

#returns a JSON object full of records to be pulled
#assumes values is a list of lists
def parse_data(values):

    set_columns(values)

    output = []
    for i, row in enumerate(values):

        if i == 0 or len(row) == 0:
            continue

        try:
            if row[PULLED] == 'done':
                continue
        except IndexError:#row[PULLED] comes back as null
            print '\t\t[identified record to pull]'

        try:
            name = "%s %s" % (row[FIRST], row[LAST])
            member = {
                'name': name,
                'birthday': parse_birthday(row[BIRTH_DAY], row[BIRTH_MONTH], row[BIRTH_YEAR]),
                'apartment': row[APT],
                'phone': row[PHONE],
                'email': row[PERSONAL_EMAIL],
                'id': i
                    }

            output.append(member)

        except IndexError:
            msg = '\tError building member object: index out of range in row %d' % i

            cprint(msg, 'red', attrs = ['bold'])
            member = {
                    'error': msg,
                    'id': i
                    }
            #TODO make an error appear in the sheet

    return json.dumps(output)

def parse_birthday(day, month, year):

    output = year

    if month == 'January':
        output += '01'
    elif month == 'February':
        output += '02'
    elif month == 'March':
        output += '03'
    elif month == 'April':
        output += '04'
    elif month == 'May':
        output += '05'
    elif month == 'June':
        output += '06'
    elif month == 'July':
        output += '07'
    elif month == 'August':
        output += '08'
    elif month == 'September':
        output += '09'
    elif month == 'October':
        output += '10'
    elif month == 'November':
        output += '11'
    elif month == 'December':
        output += '12'

    output += month

    if len(day) == 1:
        output += '0'

    output += day

    return output

def build_pulled_row(member):

    row = []

    for i in range(LENGTH):
        row.append(None)

    try:
        mrn = member['mrn']

        for n, i in enumerate(row):
            if n == MRN:
                row[n] = mrn
            elif n == PULLED:
                row[n] = 'done'

    except KeyError: #we now know it's reporting an error

        try:
            index = member['row']

            for n, i in enumerate(row):
                if n == PULLED:
                    row[n] = member['error']

        except KeyError:
            cprint('\t\tcould not find row nor mrn when building pulled row', 'red', attrs=['bold'])

    return row

def build_bishop_row(member):

    row = []

    for i in range(LENGTH):
        row.append(None)

    try:
        bishop = member['bishop']

        for n, i in enumerate(row):
            if n == BISHOP:
                row[n] = bishop

        email = member['email']

        for n, i in enumerate(row):
            if n == EMAIL:
                row[n] = email

    except KeyError: #we now know it's reporting an error
        try:
            index = member['row']

            for n, i in enumerate(row):
                if n == BISHOP:
                    row[n] = member['error']

        except KeyError:
            cprint('\t\tcould not find row nor bishop info when building pulled row', 'red', attrs=['bold'])

    return row

#@param row - list of values that goes in the row
#@param index - int representing the row
def update_row(row, index):

    index += 1

    #Can we build the service more than one time?
    service = build_service()

    range_name = 'New Members 2017!%d:%d' % (index, index)
    values = [row]
    body = {
            'values': values
            }
    service.spreadsheets().values().update(spreadsheetId = SHEET_ID, range = range_name, body = body, valueInputOption = 'RAW').execute()

#this is an example
def ex():

    service = build_service()

    range_name = 'New Members 2017!B2:C4'

    result = service.spreadsheets().values().get(spreadsheetId=SHEET_ID, range = range_name).execute()

    values = result.get('values', [])

    if not values:
        print 'no data found'
    else:
        print range_name
        for row in values:
            print row[0], row[1]

    values = [
            ['c', 'Monster'],
            ['e', 'Monster']
            ]

    body = {
            'majorDimension': 'ROWS',
            'values': values
            }

    result = service.spreadsheets().values().update(spreadsheetId=SHEET_ID, range = range_name, valueInputOption='RAW', body=body).execute()

<commit_msg>this one actually WORKS<commit_after>#!/usr/bin/env python

import httplib2
import os
import json

from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
from termcolor import cprint

SCOPES = 'https://www.googleapis.com/auth/spreadsheets'
SECRET = 'records_auth.json'
APP_NAME = 'ward_members_auth'
SHEET_ID = '13zdxFnJLJ_7IfWQxOBm0sZPkIL5bHZ0RE29o_T27Hd0'
RANGE='New Members 2017'

FIRST = -1
LAST = -1
DOB  = -1
APT = -1
PHONE = -1
PULLED = -1
MRN = -1
BISHOP = -1
EMAIL = -1
CONTACTED = -1
BIRTH_DAY = -1
BIRTH_MONTH = -1
BIRTH_YEAR = -1

LENGTH = -1

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()

except ImportError:
    flags = None

def get_credentials():

    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')

    if not os.path.exists(credential_dir):
        os.mkdirs(credential_dir)

    credential_path = os.path.join(credential_dir, 'sheets.googleapis.com-new-members.json')

    store = Storage(credential_path)
    credentials = store.get()

    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(SECRET, SCOPES)
        flow.user_agent = APP_NAME
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        print 'storing credentials to', credential_path

    return credentials

def build_service():

    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())

    discovery_url = ('https://sheets.googleapis.com/$discovery/rest?' 'version=v4')

    service = discovery.build('sheets', 'v4', http=http, discoveryServiceUrl=discovery_url)

    return service

#gets a dump of what's in Google Sheets
def get_data():

    service = build_service()

    result = service.spreadsheets().values().get(spreadsheetId = SHEET_ID, range = RANGE).execute()

    values = result.get('values', []) #what in the world does this do??

    if not values:
        raise ValueError('no data found')
    else:
        return values

#values is a list of lists
def set_columns(values):

    global LENGTH
    LENGTH = len(values[0])

    for i, cell in enumerate(values[0]):
        if cell == 'First Name':
            global FIRST
            FIRST = i
        elif cell == 'Last Name':
            global LAST
            LAST = i
        elif cell == 'Birth Month':
            global BIRTH_MONTH
            BIRTH_MONTH = i
        elif cell == 'Birth Day':
            global BIRTH_DAY
            BIRTH_DAY = i
        elif cell == 'Birth Year':
            global BIRTH_YEAR
            BIRTH_YEAR = i
        elif cell == 'Apartment':
            global APT
            APT = i
        elif cell == 'Phone Number':
            global PHONE
            PHONE = i
        elif cell == 'Email Address':
            global PERSONAL_EMAIL
            PERSONAL_EMAIL = i
        elif cell == 'Records Pulled':
            global PULLED
            PULLED = i
        elif cell == 'MRN':
            global MRN
            MRN = i
        elif cell == 'Former Bishop':
            global BISHOP
            BISHOP = i
        elif cell == 'Former Bishop Email':
            global EMAIL
            EMAIL = i
        else:
            continue

#returns a JSON object full of records to be pulled
#assumes values is a list of lists
def parse_data(values):

    set_columns(values)

    output = []
    for i, row in enumerate(values):

        if i == 0 or len(row) == 0:
            continue

        try:
            if row[PULLED] == 'done':
                continue
        except IndexError:#row[PULLED] comes back as null
            print '\t\t[identified record to pull]'

        try:
            name = "%s %s" % (row[FIRST], row[LAST])
            member = {
                'name': name,
                'birthday': parse_birthday(row[BIRTH_DAY], row[BIRTH_MONTH], row[BIRTH_YEAR]),
                'apartment': row[APT],
                'phone': row[PHONE],
                'email': row[PERSONAL_EMAIL],
                'id': i
                    }

            output.append(member)

        except IndexError:
            msg = '\tError building member object: index out of range in row %d' % i

            cprint(msg, 'red', attrs = ['bold'])
            member = {
                    'error': msg,
                    'id': i
                    }
            #TODO make an error appear in the sheet

    return json.dumps(output)

def parse_birthday(day, month, year):

    output = year

    if month == 'January':
        output += '01'
    elif month == 'February':
        output += '02'
    elif month == 'March':
        output += '03'
    elif month == 'April':
        output += '04'
    elif month == 'May':
        output += '05'
    elif month == 'June':
        output += '06'
    elif month == 'July':
        output += '07'
    elif month == 'August':
        output += '08'
    elif month == 'September':
        output += '09'
    elif month == 'October':
        output += '10'
    elif month == 'November':
        output += '11'
    elif month == 'December':
        output += '12'

    if len(day) == 1:
        output += '0'

    output += day

    return output

def build_pulled_row(member):

    row = []

    for i in range(LENGTH):
        row.append(None)

    try:
        mrn = member['mrn']

        for n, i in enumerate(row):
            if n == MRN:
                row[n] = mrn
            elif n == PULLED:
                row[n] = 'done'

    except KeyError: #we now know it's reporting an error

        try:
            index = member['row']

            for n, i in enumerate(row):
                if n == PULLED:
                    row[n] = member['error']

        except KeyError:
            cprint('\t\tcould not find row nor mrn when building pulled row', 'red', attrs=['bold'])

    return row

def build_bishop_row(member):

    row = []

    for i in range(LENGTH):
        row.append(None)

    try:
        bishop = member['bishop']

        for n, i in enumerate(row):
            if n == BISHOP:
                row[n] = bishop

        email = member['email']

        for n, i in enumerate(row):
            if n == EMAIL:
                row[n] = email

    except KeyError: #we now know it's reporting an error
        try:
            index = member['row']

            for n, i in enumerate(row):
                if n == BISHOP:
                    row[n] = member['error']

        except KeyError:
            cprint('\t\tcould not find row nor bishop info when building pulled row', 'red', attrs=['bold'])

    return row

#@param row - list of values that goes in the row
#@param index - int representing the row
def update_row(row, index):

    index += 1

    #Can we build the service more than one time?
    service = build_service()

    range_name = 'New Members 2017!%d:%d' % (index, index)
    values = [row]
    body = {
            'values': values
            }
    service.spreadsheets().values().update(spreadsheetId = SHEET_ID, range = range_name, body = body, valueInputOption = 'RAW').execute()

#this is an example
def ex():

    service = build_service()

    range_name = 'New Members 2017!B2:C4'

    result = service.spreadsheets().values().get(spreadsheetId=SHEET_ID, range = range_name).execute()

    values = result.get('values', [])

    if not values:
        print 'no data found'
    else:
        print range_name
        for row in values:
            print row[0], row[1]

    values = [
            ['c', 'Monster'],
            ['e', 'Monster']
            ]

    body = {
            'majorDimension': 'ROWS',
            'values': values
            }

    result = service.spreadsheets().values().update(spreadsheetId=SHEET_ID, range = range_name, valueInputOption='RAW', body=body).execute()

<|endoftext|>


<commit_before>import random
import datetime
import re
import os
import util
import warnings
import logging

import pandas as pd
from scipy import stats

import numpy as np
from numpy import random
from dateutil.relativedelta import relativedelta

import collections
from itertools import product

from sklearn import preprocessing, datasets
from sklearn.utils.validation import _assert_all_finite

from step import Step

class ClassificationData(Step):
    def run(self):
        X,y = datasets.make_classification(**self.__kwargs__)
        X,y = pd.DataFrame(X), pd.Series(y)

        train = np.zeros(len(X), dtype=bool)
        train[random.choice(len(X), len(X)/2)] = True
        train = pd.Series(train)

        return {'X': X, 'y': y, 'train': train, 'test': ~train}

class CreateEngine(Step):
    def run(self):
        return util.create_engine()

class FromSQL(Step):
    def __init__(self, query, **kwargs):
        Step.__init__(self, query=query, **kwargs)
        if 'inputs' not in kwargs:
            self.inputs = [CreateEngine()]

    def run(self, engine):
        return pd.read_sql(self.query, engine)

# write DataFrames to an HDF store
# pass put_arguments (format, mode, data_columns, etc.) to init
# pass DataFrames by name via inputs
class ToHDF(Step):
    def run(self, **kwargs):
        store = pd.HDFStore(os.path.join(self.get_dump_dirname(), 'result.h5'))

        for key, df in kwargs.iteritems():
            logging.info('Writing %s %s' % (key, str(df.shape)))
            if 'put_args' in self.get_arguments() and key in self.put_args:
            	args = self.put_args[key]
            else:
				args = {}

            store.put(key, df, mode='w', **args)

        return store

    def dump(self):
        return

    def load(self):
        self.set_result(pd.HDFStore(os.path.join(self.get_dump_dirname(), 'result.h5')))

class Shape(Step):
    def run(self, X, index=None, **kwargs):
        if index is not None:
            X = X[index]
        return {'n_rows': X.shape[0], 'n_cols': X.shape[1]}

class HoldOut(Step):
    def run(self, index, **kwargs):
        mask = np.zeros(len(index), dtype=bool)
        mask[random.choice(len(index), len(index)*self.p)] = True

        new_index = pd.Series(index.values & (~mask), index=index.index)
        holdout = pd.Series(index.values & mask, index=index.index)

        return {'index': new_index, 'holdout': holdout}

def percentile(series):
    return pd.Series(stats.rankdata(series)/len(series), index=series.index)

def prefix_columns(df, prefix, ignore=[]):
    df.columns =  [prefix + c if c not in ignore else c for c in df.columns]

# generate year, month, day features from specified date features
def expand_dates(df, columns=[]):
    columns=df.columns.intersection(columns)
    df2 = df.reindex(columns=set(df.columns).difference(columns))
    for column in columns:
        df2[column + '_year'] = df[column].apply(lambda x: x.year)
        df2[column + '_month'] = df[column].apply(lambda x: x.month)
        df2[column + '_day'] = df[column].apply(lambda x: x.day)
    return df2

# binarize specified categoricals
# category_classes is either a dict of (column : [class1, class2, ...]) pairs
# or a list of columns, in which case possible values are found using df[column].unique()
# all_classes = False means the last class is skipped
# drop means drop the original column
def binarize(df, category_classes, all_classes=True, drop=True):
    if type(category_classes) is not dict:
        columns = set(category_classes).intersection(df.columns)
        category_classes = {column : df[column].unique() for column in columns}
    else:
        columns = set(category_classes.keys()).intersection(df.columns)

    for category in columns:
        classes = category_classes[category]
        for i in range(len(classes)-1 if not all_classes else len(classes)):
            df[category + '_' + str(classes[i]).replace( ' ', '_')] = (df[category] == classes[i])

    if drop:
        df.drop(columns, axis=1, inplace=True)
    return df

def binarize_set(df, column, values=None):
    d = df[column].dropna() # avoid nulls
    if values is None:
        values = reduce(lambda a,b: a | b, d)
    for value in values:
        name = values[value] if type(values) is dict else str(value)
        df[column + '_'+ name.replace(' ', '_')] = d.apply(lambda c: value in c)
    df.drop(column, axis=1, inplace=True)

# given a column whose entries are lists, create columns counting each element
def count_list(df, column, values=None):
    if values is None:
        values = set(np.concatenate(df[column].values))
    for value in values:
        name = values[value] if type(values) is dict else str(value)
        df[column + '_'+ name.replace(' ', '_')] = df[column].apply(lambda d: d.count(value))
    df.drop(column, axis=1, inplace=True)

# given values, counts (as returned by np.unique(return_counts=True), find the count of a given value
def countsorted(values, counts, value):
    if len(values) == 0: # when values is empty is has float dtype and searchsorted will fail
        return 0
    i = np.searchsorted(values, value)
    if i != len(values) and values[i] == value:
        return counts[i]
    else:
        return 0

# convert (values, counts) as returned by aggregate.aggregate_counts() to dicts
# makes expand_counts much faster
def counts_to_dicts(df, column):
    d = df[column].apply(lambda c: pd.notnull(c) and len(c[0]) > 0) # index where there are counts and they aren't null
    return df.loc[d, column].apply(lambda c: {k:v for k,v in zip(*c)})

# expand a column containing value:count dictionaries
def expand_counts(df, column, values=None):
    d = counts_to_dicts(df, column)
    if len(d) > 0:
        if values is None:
            values = set(np.concatenate(d.apply(lambda c: c.keys()).values))
        for value in values:
            name = values[value] if type(values) is dict else str(value)
            df[column + '_'+ name.replace(' ', '_')] = d.apply(lambda c: c[value] if value in c else 0)
    df.drop(column, axis=1, inplace=True)

def binarize_clusters(df, column, n_clusters, train=None):
    series = df[column]
    series = series.dropna()

    from sklearn.cluster import KMeans
    kmeans = KMeans(n_clusters)

    series = pd.DataFrame(series)
    kmeans.fit(series[train] if train is not None else series)

    clusters = kmeans.cluster_centers_[:,0].astype(int)
    df[column + '_cluster'] = pd.Series(kmeans.predict(series), index=series.index).apply(lambda d: clusters[d])

    binarize(df, {column + '_cluster': clusters}, all_classes=True) # use all_classes to handle nulls

    return df

# narrows df to train | test
# then narrows train and test to that
def train_test_subset(df, train, test):
    df.drop(df.index[~(train | test)], inplace=True)
    train = train.loc[df.index]
    test = test.loc[df.index]

    return df, train, test

# returns endogenous and exogenous variables
# normalization requires imputation (can't normalize null values)
# training mask is used for normalization
def Xy(df, y_column, include=None, exclude=None, train=None, category_classes={}):
    y = df[y_column]
    exclude.add(y_column)

    X = select_features(df=df, exclude=exclude, include=include)

    X = binarize(X, category_classes)

    #nulcols = null_columns(X, train)
    #if len(nulcols) > 0:
    #    print 'Warning: null columns ' + str(nulcols)

    #nonnum = non_numeric_columns(X)
    #if len(nonnum) > 0:
    #    print 'Warning: non-numeric columns ' + str(nonnum)

    X = X.astype(np.float32, copy=False)

    #inf = infinite_columns(X)
    #if len(inf) > 0:
    #    print 'Warning: columns contain infinite values' + str(inf)

    return (X,y)

def normalize(X, train=None):
    Xfit = X[train] if train is not None else X
    sigma = Xfit.std(ddof=0)
    sigma.loc[sigma==0]=1
    mu = Xfit.mean()

    X = (X - mu) / sigma

    return X

def impute(X, train=None):
    Xfit = X[train] if train is not None else X
    X.fillna(Xfit.mean(), inplace=True)
    return X

# select subset of strings matching a regex
# treats strings as a set!
def select_regexes(strings, regexes):
    strings = set(strings)
    select = set()
    if isinstance(strings, collections.Iterable):
        for r in regexes:
            s = set(filter(re.compile('^'  + r + '$').search, strings))
            strings -= s
            select |= s
        return select
    else:
        raise ValueError("exclude should be iterable")

def exclude_regexes(strings, exclude, include=None):
    e = select_regexes(strings, exclude)
    i = select_regexes(strings, include) if include is not None else set()
    return set(strings).difference(e).union(i)

def select_features(df, exclude, include=None):
    columns = exclude_regexes(strings=df.columns, exclude=exclude, include=include)
    df = df.reindex(columns = columns, copy=False)
    return df

def null_columns(df, train=None):
    if train is not None:
        df = df[train]
    nulcols = df.isnull().sum() == len(df)
    return nulcols[nulcols==True].index

def infinite_columns(df):
    columns = []
    for c in df.columns:
        try:
            _assert_all_finite(df[c])
        except:
            columns.append(c)
    return columns

def non_numeric_columns(df):
    columns = []
    for c in df.columns:
        try:
            df[c].astype(float)
        except:
            columns.append(c)

    return columns

def get_correlates(df, c=.99):
    corr = df.corr().values
    for i in range(len(df.columns)):
        for j in range(i):
            if corr[i][j] >= c:
                print df.columns[i] + ', ' + df.columns[j] + ': ' + str(corr[i][j])

def undersample_by(y, train, p):
    a = pd.Series([random.random() < p for i in range(len(y))], index=y.index)
    return train & (y | a)

def undersample_to(y, train, p):
    T = y[train].sum()
    F = len(y[train]) - T
    q = p*T/((1-p)*F)
    return undersample_by(y, train, q)

def date_censor_sql(date_column, today, column = None):
    if column is None:
        column = date_column
    return "(CASE WHEN {date_column} < '{today}' THEN {column} ELSE null END)".format(
            date_column=date_column, today=today, column=column)

def date_select(df, date_column, date, delta):
    """
    given a series an end date and number of days, return subset in the date range
    if deta is None then there is no starting date
    """
    delta = parse_delta(delta)
    df = df[ df[date_column] < date ]

    if delta is not None:
        start_date = date - delta
        df = df[ df[date_column] >= start_date ]

    return df.copy()

def date_censor(df, date_columns, date):
    """
    a dictionary of date_column: [dependent_column1, ...] pairs
    censor the dependent columns when the date column is before the given end_date
    then censor the date column itself
    """
    for date_column, censor_columns in date_columns.iteritems():
        for censor_column in censor_columns:
            df[censor_column] = df[censor_column].where(df[date_column] < date)

        df[date_column] = df[date_column].where(df[date_column] < date)

    return df

delta_chars = {
        'y':'years', 'm':'months', 'w':'weeks', 'd':'days', 'h':'hours',
        'M':'minutes', 's':'seconds', 'u':'microseconds'
}

delta_regex = re.compile('^([0-9]+)(u|s|M|h|d|m|y)$')

# parse a string to a delta
# 'all' is represented by None
def parse_delta(s):
    if s == 'all':
        return None
    else:
        l = delta_regex.findall(s)
        if len(l) == 1:
            return relativedelta(**{delta_chars[l[0][1]]:int(l[0][0])})
        else:
            raise ValueError('Invalid delta string: %s' % s)

def nearest_neighbors_impute(df, coordinate_columns, data_columns, knr_params={}):
    from sklearn.neighbors import KNeighborsRegressor
    for column in data_columns:
        not_null = df[column].notnull()
        if (~not_null).sum() == 0:
            continue
        knr = KNeighborsRegressor(**knr_params)
        knr.fit(df.loc[not_null,coordinate_columns], df.loc[not_null,[column]])
        predicted = knr.predict(df.loc[~not_null,coordinate_columns])
        df.loc[ (~not_null),[column]] = predicted
<commit_msg>FromSql kwargs<commit_after>import random
import datetime
import re
import os
import util
import warnings
import logging

import pandas as pd
from scipy import stats

import numpy as np
from numpy import random
from dateutil.relativedelta import relativedelta

import collections
from itertools import product

from sklearn import preprocessing, datasets
from sklearn.utils.validation import _assert_all_finite

from step import Step

class ClassificationData(Step):
    def run(self):
        X,y = datasets.make_classification(**self.__kwargs__)
        X,y = pd.DataFrame(X), pd.Series(y)

        train = np.zeros(len(X), dtype=bool)
        train[random.choice(len(X), len(X)/2)] = True
        train = pd.Series(train)

        return {'X': X, 'y': y, 'train': train, 'test': ~train}

class CreateEngine(Step):
    def run(self):
        return util.create_engine()

class FromSQL(Step):
    def __init__(self, query, **kwargs):
        Step.__init__(self, query=query, **kwargs)
        if 'inputs' not in kwargs:
            self.inputs = [CreateEngine()]

    def run(self, engine):
        kwargs = dict(self.__kwargs__)
        kwargs.pop('query')
        return pd.read_sql(self.query, engine, **kwargs)

# write DataFrames to an HDF store
# pass put_arguments (format, mode, data_columns, etc.) to init
# pass DataFrames by name via inputs
class ToHDF(Step):
    def run(self, **kwargs):
        store = pd.HDFStore(os.path.join(self.get_dump_dirname(), 'result.h5'))

        for key, df in kwargs.iteritems():
            logging.info('Writing %s %s' % (key, str(df.shape)))
            if 'put_args' in self.get_arguments() and key in self.put_args:
            	args = self.put_args[key]
            else:
				args = {}

            store.put(key, df, mode='w', **args)

        return store

    def dump(self):
        return

    def load(self):
        self.set_result(pd.HDFStore(os.path.join(self.get_dump_dirname(), 'result.h5')))

class Shape(Step):
    def run(self, X, index=None, **kwargs):
        if index is not None:
            X = X[index]
        return {'n_rows': X.shape[0], 'n_cols': X.shape[1]}

class HoldOut(Step):
    def run(self, index, **kwargs):
        mask = np.zeros(len(index), dtype=bool)
        mask[random.choice(len(index), len(index)*self.p)] = True

        new_index = pd.Series(index.values & (~mask), index=index.index)
        holdout = pd.Series(index.values & mask, index=index.index)

        return {'index': new_index, 'holdout': holdout}

def percentile(series):
    return pd.Series(stats.rankdata(series)/len(series), index=series.index)

def prefix_columns(df, prefix, ignore=[]):
    df.columns =  [prefix + c if c not in ignore else c for c in df.columns]

# generate year, month, day features from specified date features
def expand_dates(df, columns=[]):
    columns=df.columns.intersection(columns)
    df2 = df.reindex(columns=set(df.columns).difference(columns))
    for column in columns:
        df2[column + '_year'] = df[column].apply(lambda x: x.year)
        df2[column + '_month'] = df[column].apply(lambda x: x.month)
        df2[column + '_day'] = df[column].apply(lambda x: x.day)
    return df2

# binarize specified categoricals
# category_classes is either a dict of (column : [class1, class2, ...]) pairs
# or a list of columns, in which case possible values are found using df[column].unique()
# all_classes = False means the last class is skipped
# drop means drop the original column
def binarize(df, category_classes, all_classes=True, drop=True):
    if type(category_classes) is not dict:
        columns = set(category_classes).intersection(df.columns)
        category_classes = {column : df[column].unique() for column in columns}
    else:
        columns = set(category_classes.keys()).intersection(df.columns)

    for category in columns:
        classes = category_classes[category]
        for i in range(len(classes)-1 if not all_classes else len(classes)):
            df[category + '_' + str(classes[i]).replace( ' ', '_')] = (df[category] == classes[i])

    if drop:
        df.drop(columns, axis=1, inplace=True)
    return df

def binarize_set(df, column, values=None):
    d = df[column].dropna() # avoid nulls
    if values is None:
        values = reduce(lambda a,b: a | b, d)
    for value in values:
        name = values[value] if type(values) is dict else str(value)
        df[column + '_'+ name.replace(' ', '_')] = d.apply(lambda c: value in c)
    df.drop(column, axis=1, inplace=True)

# given a column whose entries are lists, create columns counting each element
def count_list(df, column, values=None):
    if values is None:
        values = set(np.concatenate(df[column].values))
    for value in values:
        name = values[value] if type(values) is dict else str(value)
        df[column + '_'+ name.replace(' ', '_')] = df[column].apply(lambda d: d.count(value))
    df.drop(column, axis=1, inplace=True)

# given values, counts (as returned by np.unique(return_counts=True), find the count of a given value
def countsorted(values, counts, value):
    if len(values) == 0: # when values is empty is has float dtype and searchsorted will fail
        return 0
    i = np.searchsorted(values, value)
    if i != len(values) and values[i] == value:
        return counts[i]
    else:
        return 0

# convert (values, counts) as returned by aggregate.aggregate_counts() to dicts
# makes expand_counts much faster
def counts_to_dicts(df, column):
    d = df[column].apply(lambda c: pd.notnull(c) and len(c[0]) > 0) # index where there are counts and they aren't null
    return df.loc[d, column].apply(lambda c: {k:v for k,v in zip(*c)})

# expand a column containing value:count dictionaries
def expand_counts(df, column, values=None):
    d = counts_to_dicts(df, column)
    if len(d) > 0:
        if values is None:
            values = set(np.concatenate(d.apply(lambda c: c.keys()).values))
        for value in values:
            name = values[value] if type(values) is dict else str(value)
            df[column + '_'+ name.replace(' ', '_')] = d.apply(lambda c: c[value] if value in c else 0)
    df.drop(column, axis=1, inplace=True)

def binarize_clusters(df, column, n_clusters, train=None):
    series = df[column]
    series = series.dropna()

    from sklearn.cluster import KMeans
    kmeans = KMeans(n_clusters)

    series = pd.DataFrame(series)
    kmeans.fit(series[train] if train is not None else series)

    clusters = kmeans.cluster_centers_[:,0].astype(int)
    df[column + '_cluster'] = pd.Series(kmeans.predict(series), index=series.index).apply(lambda d: clusters[d])

    binarize(df, {column + '_cluster': clusters}, all_classes=True) # use all_classes to handle nulls

    return df

# narrows df to train | test
# then narrows train and test to that
def train_test_subset(df, train, test):
    df.drop(df.index[~(train | test)], inplace=True)
    train = train.loc[df.index]
    test = test.loc[df.index]

    return df, train, test

# returns endogenous and exogenous variables
# normalization requires imputation (can't normalize null values)
# training mask is used for normalization
def Xy(df, y_column, include=None, exclude=None, train=None, category_classes={}):
    y = df[y_column]
    exclude.add(y_column)

    X = select_features(df=df, exclude=exclude, include=include)

    X = binarize(X, category_classes)

    #nulcols = null_columns(X, train)
    #if len(nulcols) > 0:
    #    print 'Warning: null columns ' + str(nulcols)

    #nonnum = non_numeric_columns(X)
    #if len(nonnum) > 0:
    #    print 'Warning: non-numeric columns ' + str(nonnum)

    X = X.astype(np.float32, copy=False)

    #inf = infinite_columns(X)
    #if len(inf) > 0:
    #    print 'Warning: columns contain infinite values' + str(inf)

    return (X,y)

def normalize(X, train=None):
    Xfit = X[train] if train is not None else X
    sigma = Xfit.std(ddof=0)
    sigma.loc[sigma==0]=1
    mu = Xfit.mean()

    X = (X - mu) / sigma

    return X

def impute(X, train=None):
    Xfit = X[train] if train is not None else X
    X.fillna(Xfit.mean(), inplace=True)
    return X

# select subset of strings matching a regex
# treats strings as a set!
def select_regexes(strings, regexes):
    strings = set(strings)
    select = set()
    if isinstance(strings, collections.Iterable):
        for r in regexes:
            s = set(filter(re.compile('^'  + r + '$').search, strings))
            strings -= s
            select |= s
        return select
    else:
        raise ValueError("exclude should be iterable")

def exclude_regexes(strings, exclude, include=None):
    e = select_regexes(strings, exclude)
    i = select_regexes(strings, include) if include is not None else set()
    return set(strings).difference(e).union(i)

def select_features(df, exclude, include=None):
    columns = exclude_regexes(strings=df.columns, exclude=exclude, include=include)
    df = df.reindex(columns = columns, copy=False)
    return df

def null_columns(df, train=None):
    if train is not None:
        df = df[train]
    nulcols = df.isnull().sum() == len(df)
    return nulcols[nulcols==True].index

def infinite_columns(df):
    columns = []
    for c in df.columns:
        try:
            _assert_all_finite(df[c])
        except:
            columns.append(c)
    return columns

def non_numeric_columns(df):
    columns = []
    for c in df.columns:
        try:
            df[c].astype(float)
        except:
            columns.append(c)

    return columns

def get_correlates(df, c=.99):
    corr = df.corr().values
    for i in range(len(df.columns)):
        for j in range(i):
            if corr[i][j] >= c:
                print df.columns[i] + ', ' + df.columns[j] + ': ' + str(corr[i][j])

def undersample_by(y, train, p):
    a = pd.Series([random.random() < p for i in range(len(y))], index=y.index)
    return train & (y | a)

def undersample_to(y, train, p):
    T = y[train].sum()
    F = len(y[train]) - T
    q = p*T/((1-p)*F)
    return undersample_by(y, train, q)

def date_censor_sql(date_column, today, column = None):
    if column is None:
        column = date_column
    return "(CASE WHEN {date_column} < '{today}' THEN {column} ELSE null END)".format(
            date_column=date_column, today=today, column=column)

def date_select(df, date_column, date, delta):
    """
    given a series an end date and number of days, return subset in the date range
    if deta is None then there is no starting date
    """
    delta = parse_delta(delta)
    df = df[ df[date_column] < date ]

    if delta is not None:
        start_date = date - delta
        df = df[ df[date_column] >= start_date ]

    return df.copy()

def date_censor(df, date_columns, date):
    """
    a dictionary of date_column: [dependent_column1, ...] pairs
    censor the dependent columns when the date column is before the given end_date
    then censor the date column itself
    """
    for date_column, censor_columns in date_columns.iteritems():
        for censor_column in censor_columns:
            df[censor_column] = df[censor_column].where(df[date_column] < date)

        df[date_column] = df[date_column].where(df[date_column] < date)

    return df

delta_chars = {
        'y':'years', 'm':'months', 'w':'weeks', 'd':'days', 'h':'hours',
        'M':'minutes', 's':'seconds', 'u':'microseconds'
}

delta_regex = re.compile('^([0-9]+)(u|s|M|h|d|m|y)$')

# parse a string to a delta
# 'all' is represented by None
def parse_delta(s):
    if s == 'all':
        return None
    else:
        l = delta_regex.findall(s)
        if len(l) == 1:
            return relativedelta(**{delta_chars[l[0][1]]:int(l[0][0])})
        else:
            raise ValueError('Invalid delta string: %s' % s)

def nearest_neighbors_impute(df, coordinate_columns, data_columns, knr_params={}):
    from sklearn.neighbors import KNeighborsRegressor
    for column in data_columns:
        not_null = df[column].notnull()
        if (~not_null).sum() == 0:
            continue
        knr = KNeighborsRegressor(**knr_params)
        knr.fit(df.loc[not_null,coordinate_columns], df.loc[not_null,[column]])
        predicted = knr.predict(df.loc[~not_null,coordinate_columns])
        df.loc[ (~not_null),[column]] = predicted
<|endoftext|>


<commit_before># MIT License
#
# Copyright (c) 2017 Jonatan Almén, Alexander Håkansson, Jesper Jaxing, Gmal
# Tchaefa, Maxim Goretskyy, Axel Olivecrona
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#==============================================================================
"""
A module for handling training, validation and test data for the ANN model
"""
import csv_reader
import helper

FILE_TRAINING = "training_data.csv"
FILE_VALIDATION = "validation_data.csv"
FILE_TESTING = "testing_data.csv"
DEFAULT_BATCH_SIZE = 1

class Data(object):
    """ A class for getting handling data """
    def __init__(self, data_path="./data/", verbose=False):
        self._data_path = data_path
        self._verbose = verbose
        self._current_train_index = 0
        self.completed_training_epochs = 0
        self.percent_of_epoch = 0.0
        self._read_data()
        self._build_dict()

    def _read_data(self):
        """ Reads all the data from specified path """
        if self._verbose:
            print("Reading training data...")
        self.train_data, self.train_labels = \
            csv_reader.read(self._data_path + FILE_TRAINING, [0], 1)
        if self._verbose:
            print("Reading validation data...")
        self.valid_data, self.valid_labels = \
            csv_reader.read(self._data_path + FILE_VALIDATION, [0], 1)
        if self._verbose:
            print("Reading testing data...")
        self.test_data, self.test_labels = \
            csv_reader.read(self._data_path + FILE_TESTING, [0], 1)

    def _build_dict(self):
        """ Builds dictionaries using given data """
        if self._verbose:
            print("Building dictionaries...")
        vocab = " ".join(self.train_data).split()
        users = " ".join(self.train_labels).split()

        _, _, self.word_dict, self.rev_dict = helper.build_dataset(vocab)
        _, _, self.users_dict, self.rev_users_dict = helper.build_dataset(users)

    def next_train_batch(self, title_length, user_count,
                         batch_size=DEFAULT_BATCH_SIZE):
        """ Get the next batch of training data """
        batch_x = []
        batch_y = []
        for _ in range(0, batch_size):
            sentence = self.train_data[self._current_train_index]
            label = self.train_labels[self._current_train_index]
            self._current_train_index += 1

            # Support multiple epochs
            if self._current_train_index >= len(self.train_data):
                self._current_train_index = 0
                self.completed_training_epochs += 1
                self.percent_of_epoch = 0.0

            # Turn sentences and labels into vector representations
            sentence_vec = helper.get_indicies(sentence,
                                               self.word_dict, title_length)
            label_vec = helper.label_vector(label.split(), self.users_dict,
                                            user_count)
            batch_x.append(sentence_vec)
            batch_y.append(label_vec)

        self.percent_of_epoch = self._current_train_index / len(self.train_data)
        return batch_x, batch_y

<commit_msg>Pre-calculates size of training, validation and test data<commit_after># MIT License
#
# Copyright (c) 2017 Jonatan Almén, Alexander Håkansson, Jesper Jaxing, Gmal
# Tchaefa, Maxim Goretskyy, Axel Olivecrona
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#==============================================================================
"""
A module for handling training, validation and test data for the ANN model
"""
import csv_reader
import helper

FILE_TRAINING = "training_data.csv"
FILE_VALIDATION = "validation_data.csv"
FILE_TESTING = "testing_data.csv"
DEFAULT_BATCH_SIZE = 1

class Data(object):
    """ A class for getting handling data """
    def __init__(self, data_path="./data/", verbose=False):
        self._data_path = data_path
        self._verbose = verbose
        self._current_train_index = 0
        self.completed_training_epochs = 0
        self.percent_of_epoch = 0.0
        self._read_data()
        self._build_dict()

    def _read_data(self):
        """ Reads all the data from specified path """
        if self._verbose:
            print("Reading training data...")
        self.train_data, self.train_labels = \
            csv_reader.read(self._data_path + FILE_TRAINING, [0], 1)
        self.train_size = len(self.train_data)
        if self._verbose:
            print("Reading validation data...")
        self.valid_data, self.valid_labels = \
            csv_reader.read(self._data_path + FILE_VALIDATION, [0], 1)
        self.valid_size = len(self.valid_data)
        if self._verbose:
            print("Reading testing data...")
        self.test_data, self.test_labels = \
            csv_reader.read(self._data_path + FILE_TESTING, [0], 1)
        self.test_size = len(self.test_data)

    def _build_dict(self):
        """ Builds dictionaries using given data """
        if self._verbose:
            print("Building dictionaries...")
        vocab = " ".join(self.train_data).split()
        users = " ".join(self.train_labels).split()

        _, _, self.word_dict, self.rev_dict = helper.build_dataset(vocab)
        _, _, self.users_dict, self.rev_users_dict = helper.build_dataset(users)

    def next_train_batch(self, title_length, user_count,
                         batch_size=DEFAULT_BATCH_SIZE):
        """ Get the next batch of training data """
        batch_x = []
        batch_y = []
        for _ in range(0, batch_size):
            sentence = self.train_data[self._current_train_index]
            label = self.train_labels[self._current_train_index]
            self._current_train_index += 1

            # Support multiple epochs
            if self._current_train_index >= self.train_size:
                self._current_train_index = 0
                self.completed_training_epochs += 1
                self.percent_of_epoch = 0.0

            # Turn sentences and labels into vector representations
            sentence_vec = helper.get_indicies(sentence,
                                               self.word_dict, title_length)
            label_vec = helper.label_vector(label.split(), self.users_dict,
                                            user_count)
            batch_x.append(sentence_vec)
            batch_y.append(label_vec)

        self.percent_of_epoch = self._current_train_index / self.train_size
        return batch_x, batch_y

<|endoftext|>


<commit_before>from __future__ import division

import pandas as pd

from plotting import NUMERIC_TYPES

def get_numeric_features(df):
    float_feature_names = [feature for feature in df.columns
                           if df[feature].dtype in NUMERIC_TYPES]
    return df[float_feature_names]

def arff_to_df(arff):
    rows = []
    for row in arff[0]:
        rows.append(list(row))
    attributes = [x for x in arff[1]]
    return pd.DataFrame(rows, columns=attributes)

def get_nominal_integer_dict(nominal_vals):
    """
    Takes a set of nominal values (non-numeric)
    and returns a dictionary that maps each value
    to a unique integer
    """
    d = {}
    for val in nominal_vals:
        if val not in d:
            current_max = max(d.values()) if len(d) > 0 else -1
            d[val] = current_max+1
    return d

def convert_to_integer(srs, return_dict=False):
    """
    Convert a series of nominal values
    into a series of integers
    """
    d = get_nominal_integer_dict(srs)
    result =  srs.map(lambda x: d[x])
    if return_dict:
        return (result, d)
    else:
        return result

def convert_strings_to_integer(df):
    ret = pd.DataFrame()
    for column_name in df:
        column = df[column_name]
        if column.dtype not in NUMERIC_TYPES:
            ret[column_name] = convert_to_integer(column)
        else:
            ret[column_name] = column
    return ret

def get_index_rows(srs, indices):
    """
    Given a dataframe and a list of indices,
    return a list of row indices corresponding
    to the supplied indices (maintaining order)
    """
    rows = []
    for i, (index, row) in enumerate(srs.iteritems()):
        if index in indices:
            rows.append(i)
    return rows

<commit_msg>Add functional data manipulation methods<commit_after>from __future__ import division

import pandas as pd

from plotting import NUMERIC_TYPES

def exclude(df, exclude):
    """
    Return a DataFrame with a set of feature excluded
    """
    keep = [column for column in df.columns
            if column not in exclude]
    return df[keep]

def partition(df, partition_function):
    """
    Separate a dataframe row-wise based on the
    value of the supplied partition function
    """

    partition_values = df.apply(partition_function, axis=1)
    return df.groupby(partition_values)

def split(df, columns):
    """
    Separate a dataframe column-wise and turn a tuple
    of all the given columns on one side and the remaining
    columns on the other side
    """

    left = columns
    right = [column for column in df.columns
             if column not in left]

    return (df[left], df[right])
    #partition_values = df.apply(partition_function, axis=1)
    #return df.groupby(partition_values)

def get_numeric_features(df):
    float_feature_names = [feature for feature in df.columns
                           if df[feature].dtype in NUMERIC_TYPES]
    return df[float_feature_names]

def arff_to_df(arff):
    rows = []
    for row in arff[0]:
        rows.append(list(row))
    attributes = [x for x in arff[1]]
    return pd.DataFrame(rows, columns=attributes)

def get_nominal_integer_dict(nominal_vals):
    """
    Takes a set of nominal values (non-numeric)
    and returns a dictionary that maps each value
    to a unique integer
    """
    d = {}
    for val in nominal_vals:
        if val not in d:
            current_max = max(d.values()) if len(d) > 0 else -1
            d[val] = current_max+1
    return d

def convert_to_integer(srs, return_dict=False):
    """
    Convert a series of nominal values
    into a series of integers
    """
    d = get_nominal_integer_dict(srs)
    result =  srs.map(lambda x: d[x])
    if return_dict:
        return (result, d)
    else:
        return result

def convert_strings_to_integer(df):
    ret = pd.DataFrame()
    for column_name in df:
        column = df[column_name]
        if column.dtype not in NUMERIC_TYPES:
            ret[column_name] = convert_to_integer(column)
        else:
            ret[column_name] = column
    return ret

def get_index_rows(srs, indices):
    """
    Given a dataframe and a list of indices,
    return a list of row indices corresponding
    to the supplied indices (maintaining order)
    """
    rows = []
    for i, (index, row) in enumerate(srs.iteritems()):
        if index in indices:
            rows.append(i)
    return rows

<|endoftext|>