<commit_before>
'''
This module should be organized as follows:

Main function:
chi_estimate() = returns chi_n, chi_b
    - calls:
        wealth.get_wealth_data() - returns data moments on wealth distribution
        labor.labor_data_moments() - returns data moments on labor supply
        minstat() - returns min of statistical objective function
            model_moments() - returns model moments
                SS.run_SS() - return SS distributions

'''

'''
------------------------------------------------------------------------
Last updated: 4/11/2016

Uses a simulated method of moments to calibrate the chi_n adn chi_b
parameters of OG-USA.

This py-file calls the following other file(s):
    wealth.get_wealth_data()
    labor.labor_data_moments()
    SS.run_SS

This py-file creates the following other file(s): None
------------------------------------------------------------------------
'''

import numpy as np
import scipy.optimize as opt
import wealth
import labor
import SS

def chi_estimate(income_tax_params, ss_params, iterative_params, chi_guesses, baseline_dir="./OUTPUT"):

    # unpack tuples of parameters
    J, S, T, BW, beta, sigma, alpha, Z, delta, ltilde, nu, g_y,\
                  g_n_ss, tau_payroll, tau_bq, rho, omega_SS, lambdas, \
                  imm_rates, e, retire, mean_income_data, h_wealth, p_wealth,\
                  m_wealth, b_ellipse, upsilon = ss_params
    chi_b_guess, chi_n_guess = chi_guesses

    flag_graphs = False

    # Generate Wealth data moments
    wealth_moments = wealth.get_wealth_data(lambdas, J, flag_graphs)

    # Generate labor data moments
    labor_moments = labor.labor_data_moments(flag_graphs,baseline_dir)

    # combine moments
    data_moments = list(wealth_moments.flatten()) + list(labor_moments.flatten())

    # call minimizer
    bnds = np.tile(np.array([1e-12, None]),(S+J,1)) # Need (1e-12, None) S+J times
    #min_args = () data_moments
    chi_guesses_flat = list(chi_b_guess.flatten()) + list(chi_n_guess.flatten())
    min_args = data_moments, income_tax_params, ss_params, iterative_params, chi_guesses_flat, baseline_dir
    print 'size of chi: ', len(chi_guesses_flat)
    print 'bnds size: ', bnds.shape
    est_output = opt.minimize(minstat, chi_guesses_flat, args=(min_args), method="L-BFGS-B", bounds=bnds,
                    tol=1e-15)
    chi_params = est_output.x
    objective_func_min = est_out.fun

    return chi_params

def minstat(chi_guesses, *args):
    '''
    --------------------------------------------------------------------
    This function generates the weighted sum of squared differences
    between the model and data moments.

    RETURNS: wssqdev
    --------------------------------------------------------------------
    '''

    data_moments, income_tax_params, ss_params, iterative_params, chi_params, baseline_dir = args
    J, S, T, BW, beta, sigma, alpha, Z, delta, ltilde, nu, g_y,\
                  g_n_ss, tau_payroll, tau_bq, rho, omega_SS, lambdas, \
                  imm_rates, e, retire, mean_income_data, h_wealth, p_wealth,\
                  m_wealth, b_ellipse, upsilon = ss_params
    chi_b = chi_guesses[:J]
    chi_n = chi_guesses[J:]
    chi_params = (chi_b, chi_n)
    ss_output = SS.run_SS(income_tax_params, ss_params, iterative_params, chi_params, True, baseline_dir)

    model_moments = calc_moments(ss_output, lambdas, S, J)

    # distance with levels
    distance = ((np.array(model_moments) - np.array(data_moments))**2).sum()
    print 'DATA and MODEL DISTANCE: ', distance

    # # distance with percentage diffs
    # distance = (((model_moments - data_moments)/data_moments)**2).sum()

    return distance

def calc_moments(ss_output, lambdas, S, J):
    '''
    --------------------------------------------------------------------
    This function calculates moments from the SS output that correspond
    to the data moments used for estimation.

    RETURNS: model_moments
    --------------------------------------------------------------------
    '''

    wealth_model = ss_output['bssmat']
    factor_model = ss_output['factor_ss']

    # wealth moments
    # Convert wealth levels from model to dollar terms
    wealth_model_dollars = wealth_model * factor_model
    model_wealth_moments = np.zeros(2 * J)
    # Look at the percent difference between the fits for the first age group (20-44) and second age group (45-65)
    #   The wealth_data_array moment indices are shifted because they start at age 18
    # The :: indices is so that we look at the 2 moments for the lowest group,
    # the 2 moments for the second lowest group, etc in order
    model_wealth_moments[0::2] = np.mean(wealth_model_dollars[:24],axis=0)
    model_wealth_moments[1::2] = np.mean(wealth_model_dollars[24:45],axis=0)

    # labor moments
    n = ss_output['nssmat']
    model_labor_moments = (n.reshape(S, J) * lambdas.reshape(1, J)).sum(axis=1)

    # combine moments
    model_moments = list(model_wealth_moments.flatten()) + list(model_labor_moments.flatten())

    return model_moments

<commit_msg>gini for model output<commit_after>
'''
This module should be organized as follows:

Main function:
chi_estimate() = returns chi_n, chi_b
    - calls:
        wealth.get_wealth_data() - returns data moments on wealth distribution
        labor.labor_data_moments() - returns data moments on labor supply
        minstat() - returns min of statistical objective function
            model_moments() - returns model moments
                SS.run_SS() - return SS distributions

'''

'''
------------------------------------------------------------------------
Last updated: 4/11/2016

Uses a simulated method of moments to calibrate the chi_n adn chi_b
parameters of OG-USA.

This py-file calls the following other file(s):
    wealth.get_wealth_data()
    labor.labor_data_moments()
    SS.run_SS

This py-file creates the following other file(s): None
------------------------------------------------------------------------
'''

import numpy as np
import scipy.optimize as opt
import wealth
import labor
import SS

def chi_estimate(income_tax_params, ss_params, iterative_params, chi_guesses, baseline_dir="./OUTPUT"):

    # unpack tuples of parameters
    J, S, T, BW, beta, sigma, alpha, Z, delta, ltilde, nu, g_y,\
                  g_n_ss, tau_payroll, tau_bq, rho, omega_SS, lambdas, \
                  imm_rates, e, retire, mean_income_data, h_wealth, p_wealth,\
                  m_wealth, b_ellipse, upsilon = ss_params
    chi_b_guess, chi_n_guess = chi_guesses

    flag_graphs = False

    # Generate Wealth data moments
    wealth_moments = wealth.get_wealth_data(lambdas, J, flag_graphs)

    # Generate labor data moments
    labor_moments = labor.labor_data_moments(flag_graphs,baseline_dir)

    # combine moments
    data_moments = list(wealth_moments.flatten()) + list(labor_moments.flatten())

    # call minimizer
    bnds = np.tile(np.array([1e-12, None]),(S+J,1)) # Need (1e-12, None) S+J times
    #min_args = () data_moments
    chi_guesses_flat = list(chi_b_guess.flatten()) + list(chi_n_guess.flatten())
    min_args = data_moments, income_tax_params, ss_params, iterative_params, chi_guesses_flat, baseline_dir
    print 'size of chi: ', len(chi_guesses_flat)
    print 'bnds size: ', bnds.shape
    est_output = opt.minimize(minstat, chi_guesses_flat, args=(min_args), method="L-BFGS-B", bounds=bnds,
                    tol=1e-15)
    chi_params = est_output.x
    objective_func_min = est_out.fun

    return chi_params

def minstat(chi_guesses, *args):
    '''
    --------------------------------------------------------------------
    This function generates the weighted sum of squared differences
    between the model and data moments.

    RETURNS: wssqdev
    --------------------------------------------------------------------
    '''

    data_moments, income_tax_params, ss_params, iterative_params, chi_params, baseline_dir = args
    J, S, T, BW, beta, sigma, alpha, Z, delta, ltilde, nu, g_y,\
                  g_n_ss, tau_payroll, tau_bq, rho, omega_SS, lambdas, \
                  imm_rates, e, retire, mean_income_data, h_wealth, p_wealth,\
                  m_wealth, b_ellipse, upsilon = ss_params
    chi_b = chi_guesses[:J]
    chi_n = chi_guesses[J:]
    chi_params = (chi_b, chi_n)
    ss_output = SS.run_SS(income_tax_params, ss_params, iterative_params, chi_params, True, baseline_dir)

    model_moments = calc_moments(ss_output, lambdas, S, J)

    # distance with levels
    distance = ((np.array(model_moments) - np.array(data_moments))**2).sum()
    print 'DATA and MODEL DISTANCE: ', distance

    # # distance with percentage diffs
    # distance = (((model_moments - data_moments)/data_moments)**2).sum()

    return distance

def calc_moments(ss_output, lambdas, S, J):
    '''
    --------------------------------------------------------------------
    This function calculates moments from the SS output that correspond
    to the data moments used for estimation.

    RETURNS: model_moments
    --------------------------------------------------------------------
    '''

    wealth_model = ss_output['bssmat']
    factor_model = ss_output['factor_ss']
    pop_dist = ss_output['omega_SS']

    # wealth moments
    print 'Start the ginis:\n'
    gini_coeff = gini_nocol(wealth_model, pop_dist)

    # Convert wealth levels from model to dollar terms
    wealth_model_dollars = wealth_model * factor_model
    model_wealth_moments = np.zeros(2 * J)
    # Look at the percent difference between the fits for the first age group (20-44) and second age group (45-65)
    #   The wealth_data_array moment indices are shifted because they start at age 18
    # The :: indices is so that we look at the 2 moments for the lowest group,
    # the 2 moments for the second lowest group, etc in order
    model_wealth_moments[0::2] = np.mean(wealth_model_dollars[:24],axis=0)
    model_wealth_moments[1::2] = np.mean(wealth_model_dollars[24:45],axis=0)

    # labor moments
    n = ss_output['nssmat']
    model_labor_moments = (n.reshape(S, J) * lambdas.reshape(1, J)).sum(axis=1)

    # combine moments
    model_moments = list(model_wealth_moments.flatten()) + list(model_labor_moments.flatten())

    return model_moments

def gini_nocol(path, omega):
    '''
    --------------------------------------------------------------------
    This function calculates the gini coefficient for inputs formatted like
    the model output: dimensions of (T,S,J) and a weight matrix of the same
    dimension.

    RETURNS: gini coeff
    --------------------------------------------------------------------
    '''

    pathx = np.copy(path)
    mask = pathx < 0
    pathx[mask] = 0
    pathx = pathx.reshape(T, S*J)
    total = pathx.sum(1)
    pathx /= total.reshape(T, 1)
    omega = omega.reshape(T, S*J)
    idx = np.argsort(pathx-omega, axis=1)
    path2 = pathx[:, idx][np.eye(T,T, dtype=bool)]
    omega_sorted = omega[:, idx][np.eye(T,T, dtype=bool)]
    cum_levels = np.zeros((T, S*J))
    cum_omega = np.zeros((T, S*J))
    cum_omega[:, 0] = omega_sorted[:, 0]
    cum_levels[:, 0] = path2[:, 0]
    for i in xrange(1, S*J):
        cum_levels[:, i] = path2[:, i] + cum_levels[:, i-1]
        cum_omega[:, i] = omega_sorted[:, i] + cum_omega[:, i-1]
    cum_levels[:, 1:] = cum_levels[:, :-1]
    cum_levels[:, 0] = 0
    G = 2 * (.5-(cum_levels * omega_sorted).sum(1))
    print G[-1]
    return G

<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
import re
import sys

longest_sequence = 1601

minlen = 45

hits = {}

with open('/home/redwards/Desktop/16S_all_padded.blastn', 'r') as f:
    for l in f:
        p = l.strip().split('\t')
        for i in [2,10,11]:
            p[i] = float(p[i])
        for i in range(3,10):
            p[i] = int(p[i])

        if p[8] > p[9]:
            (p[8], p[9]) = (p[9], p[8])

        m = re.match('fig\|(\d+\.\d+)', p[1])
        if not m:
            sys.stderr.write("Can't parse " + p[1] + "\n")
            continue
        sid = m.group(1)

        if sid not in hits:
            hits[sid]=[0] * longest_sequence

        for i in range(p[8], p[9]+1):
            hits[sid][i] = hits[sid][i] + 1

fig = plt.figure()
ax = fig.add_subplot(111)

x=range(longest_sequence)

for sid in hits:
    ax.plot(x, hits[sid], label=sid)

ax.legend()
fig.set_facecolor('white')
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()

plt.show()

<commit_msg>Updating<commit_after>import matplotlib.pyplot as plt
import re
import sys

longest_sequence = 1601

minlen = 45

hits = {}

with open('/home/redwards/Desktop/16S_all_padded.blastn', 'r') as f:
    for l in f:
        p = l.strip().split('\t')
        for i in [2,10,11]:
            p[i] = float(p[i])
        for i in range(3,10):
            p[i] = int(p[i])

        if p[8] > p[9]:
            (p[8], p[9]) = (p[9], p[8])

        m = re.match('fig\|(\d+\.\d+)', p[1])
        if not m:
            sys.stderr.write("Can't parse " + p[1] + "\n")
            continue
        sid = m.group(1)

        if sid not in hits:
            hits[sid]=[0] * longest_sequence

        for i in range(p[8], p[9]+1):
            hits[sid][i] = hits[sid][i] + 1

fig = plt.figure()
ax = fig.add_subplot(111)

x=range(longest_sequence)

for sid in hits:
    ax.plot(x, hits[sid], label=sid)

ax.legend()
fig.set_facecolor('white')
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()

plt.show()

<|endoftext|>


<commit_before>import openpyxl
import xlrd
import xlsxwriter
import lxml
from lxml import html
import requests

##################################################################################################################

class CardCollection:

    def __init__(self,collectionFile):

    #  create member variable self.workingDir to hold directory to use Could either ask who's running the program
    #  create member variable self.collection to hold filename and set it here----> is this done? priority?
    # is this something worth doing? priority is low
        self.collectionFile = collectionFile
        self.sourceWorkbook = xlrd.open_workbook(self.collectionFile)

    def getNumSheets(self):
        # delete when member variable self.collection created
        numb_sheets = self.sourceWorkbook.nsheets
        return numb_sheets

    def getNumRows(self, sheetIndex):
        sheet_dest = self.sourceWorkbook.sheet_by_index(sheetIndex)
        numb_rows = sheet_dest.nrows - 1
        return numb_rows

    def getSheetName(self,sheetIndex):
        sh_names_list= self.sourceWorkbook.sheet_names()
        sh_name = sh_names_list[sheetIndex]
        return sh_name

    def getCardName(self,sheetIndex,rowIndex):

        my_sheet = self.sourceWorkbook.sheet_by_index(sheetIndex)
        my_cardname = my_sheet.cell(rowIndex,1).value
        return my_cardname

    def getAllCardInfo(self,sheetIndex,rowIndex):

        cardInfo = []
        cardInfo.append(self.getCardName(sheetIndex,rowIndex))
        #print ('list of card info under getAllCardinfo: ', cardInfo)

        my_sheet = self.sourceWorkbook.sheet_by_index(sheetIndex)
        #this  gets 5 sequential items
        for col in range(2,7):
        	# xxx find a way to convert number to string 1.0 to 1
            cardInfo.append(my_sheet.cell(rowIndex,col).value)
        # gets location
        cardInfo.append(my_sheet.cell(rowIndex,11).value)
        #print(cardInfo)
        return cardInfo

class WebScraperMTG:

    # Private members
    # self.webpage
    # self.card_list
    # self.price_list

    # Private methods
    # setInformationLists()

    # Public methods
    # __init__(webpage)
    # getWebInformationList(cardname) - returns a list

    def __init__(self,webpage):

        self.webpage = webpage
        self.price_list = []
        self.card_list = []
        self.setInformationLists()

    def setInformationLists(self):

        page = requests.get(self.webpage)
        #below will pars the contents with lxml
        tree = html.fromstring(page.content)
        #below will select elements we need-return a list of elements
        card_list = tree.xpath('//a[@data-full-image]/text()')

        size = int(len(card_list) / 2)
        self.card_list = card_list[0:size]
        print(self.card_list)

        price_list = tree.xpath('//td[@class="text-right"]/text()')

        price_list = [x for x in price_list if x != '\n']
        size = int(len(price_list)/2)
        self.price_list = []
        for i in range(0,size,3):
            value = price_list[i]
            value = value[1:-1]
            self.price_list.append(value)

     #           self.daily_change_list = []
     #     for j in range(1,size,3):
     #               value = price_list[j]
     #             value = value[1:-1]
     #      self.daily_change_list.append(value)
     #    newSize = int(len(self.daily_change_list))
     #       self.daily_change_list = self.daily_change_list[0:newSize]
     #      print(newSize)
     #       print(self.daily_change_list)

     #    self.weekly_change_list = [] for k in range(2,size,3):     value = price_list[j]     value = value[1:-1]
     #        self.weekly_change_list.append(value) newSize = int(len(self.weekly_change_list)) self.weekly_change_list =
     #    self.weekly_change_list[0:newSize] print(newSize) print(self.weekly_change_list)

    #def getWebInformationList(self,cardname):
    #def getWebPriceList(self):
     #   return self.price_list

class CardSummary:

    #def __init__(self):

    def setHeader(self):
        self.currentrow = 0
        header_list = ['Card name','Color','Rarity','Foil','Special','Number','Location','Price']
        self.open_workbook = openpyxl.load_workbook('final_magic.xlsx')
        self.currentsheet = self.open_workbook.active
        # if first sheet is included in new workbook, write there
        for item in range(7):
            self.currentsheet.append([header_list[item]])
            self.open_workbook.save('final_magic.xlsx')

        print('finished setting first row, once for all')

    # Writes each merged combined line from card collection and web page to output file
    # Saves output file
    def writeSummaryRow(self):
    	# rowindex+1  , culomn =0 ?
    	# read from summary list from first item (loop through)
        self.currentrow += 1

    def saveCardInfo(self):  # transfer this to CardSummary class and adjust it

    	# create a new workbook-only once
        # if final_magic sheet does not exist create one:
        new_workbook = xlsxwriter.Workbook('final_magic.xlsx')
        # add one sheet and assign a name-only once
        new_sheet = new_workbook.add_worksheet('NewZen')

        i = 0

        # adding header -only once
        for item in cardInfo:
            xwrite = new_sheet.write(1,i,cardInfo[i])
            i += 1
        new_workbook.close()
        print('Card saved to new excel file')

###################################################################################################################

# Card Collection Test Suite
# implement the test suite with nose
oCC = CardCollection('/Users/mity/mypy/MTG_Collection_4_20_16.xlsx')
numSheets = oCC.getNumSheets()
numRows = oCC.getNumRows(46)
sheetName = oCC.getSheetName(46)
cardName = oCC.getCardName(46,1)
cardInfo = oCC.getAllCardInfo(46,1)

oCS = CardSummary()
output_file = oCS.setHeader()

#oWS = WebScraperMTG('http://www.mtggoldfish.com/index/ZEN#paper')
#priceList = oWS.getWebPriceList()

#print("Price List from object: ", priceList)

print('Total number of SHEETS in this file is:', numSheets)
print('Total number of ROWS in this sheet is:',numRows)
print('The SHEET NAME found is:',sheetName)
print('The CARD NAME is:',cardName)
print('The list of all required information is:', cardInfo)

# if numSheets == 67:
#     print("numSheets Check = TRUE")
# else:
#     ##print("numSheets Check = FALSE")
#     print("numsheet reported is FALSE, expected value is" , numSheets)

# if numRows == 323:
#     print("numRows Check = TRUE")
# else:
#     print("numRows Check = FALSE")

# if sheetName == "ZEN":
#     print("sheetName Check = TRUE")
# else:
#     print("sheetName Check = FALSE")

# if cardName == "Bala Ged Thief":
#     print("cardName Check = TRUE")
# else:
#     print("cardName Check = FALSE")

# if cardInfo[0] == 'Bala Ged Thief':
#     print("Cardname Check from list = TRUE")
# else:
#     print("Cardname Check from list = FALSE")

# if cardInfo[1] == 'B':
#      print("Color Check from list = TRUE")
# else:
#      print("Color Check from list = FALSE")

# if cardInfo[2] == 'Rare':
#     print("Rarity Check from list= TRUE")
# else:
#     print("Rarity Check from list=  FALSE")

# if cardInfo[3] == 'N':
#     print("Foil Check from list = TRUE")
# else:
#     print("Foil Check from list = FALSE")

# if cardInfo[4] == 'N':
#     print("Special Check from list = TRUE")
# else:
#     print("Special Check from list = FALSE")

# if cardInfo[5] == '1':
#     print("Number Check from list = TRUE")
# else:
#     print("Number from list = FALSE")

# if cardInfo[6] == 'Zendikar - Storage Box':
#     print("Location Check from list = TRUE")
# else:
#     print("Location Check from list = FALSE")
# cardName = oCC.getCardname(47,5)
# cardInfo = oCC.getAllCardInfo(47,5)

# print(cardName)
# if cardName == "Bloodchief Ascension":
#     print("cardname Check = TRUE")
# else:
#     print("cardname Check = FALSE")
<commit_msg>setheader works<commit_after>import openpyxl
import xlrd
import xlsxwriter
import lxml
from lxml import html
import requests

##################################################################################################################

class CardCollection:

    def __init__(self,collectionFile):

    #  create member variable self.workingDir to hold directory to use Could either ask who's running the program
    #  create member variable self.collection to hold filename and set it here----> is this done? priority?
    # is this something worth doing? priority is low
        self.collectionFile = collectionFile
        self.sourceWorkbook = xlrd.open_workbook(self.collectionFile)

    def getNumSheets(self):
        # delete when member variable self.collection created
        numb_sheets = self.sourceWorkbook.nsheets
        return numb_sheets

    def getNumRows(self, sheetIndex):
        sheet_dest = self.sourceWorkbook.sheet_by_index(sheetIndex)
        numb_rows = sheet_dest.nrows - 1
        return numb_rows

    def getSheetName(self,sheetIndex):
        sh_names_list= self.sourceWorkbook.sheet_names()
        sh_name = sh_names_list[sheetIndex]
        return sh_name

    def getCardName(self,sheetIndex,rowIndex):

        my_sheet = self.sourceWorkbook.sheet_by_index(sheetIndex)
        my_cardname = my_sheet.cell(rowIndex,1).value
        return my_cardname

    def getAllCardInfo(self,sheetIndex,rowIndex):

        cardInfo = []
        cardInfo.append(self.getCardName(sheetIndex,rowIndex))
        #print ('list of card info under getAllCardinfo: ', cardInfo)

        my_sheet = self.sourceWorkbook.sheet_by_index(sheetIndex)
        #this  gets 5 sequential items
        for col in range(2,7):
        	# XXXX find a way to convert number to string 1.0 to 1
            cardInfo.append(my_sheet.cell(rowIndex,col).value)
        # gets location
        cardInfo.append(my_sheet.cell(rowIndex,11).value)
        #print(cardInfo)
        return cardInfo

class WebScraperMTG:

    # Private members
    # self.webpage
    # self.card_list
    # self.price_list

    # Private methods
    # setInformationLists()

    # Public methods
    # __init__(webpage)
    # getWebInformationList(cardname) - returns a list

    def __init__(self,webpage):

        self.webpage = webpage
        self.price_list = []
        self.card_list = []
        self.setInformationLists()

    def setInformationLists(self):

        page = requests.get(self.webpage)
        #below will pars the contents with lxml
        tree = html.fromstring(page.content)
        #below will select elements we need-return a list of elements
        card_list = tree.xpath('//a[@data-full-image]/text()')

        size = int(len(card_list) / 2)
        self.card_list = card_list[0:size]
        print(self.card_list)

        price_list = tree.xpath('//td[@class="text-right"]/text()')

        price_list = [x for x in price_list if x != '\n']
        size = int(len(price_list)/2)
        self.price_list = []
        for i in range(0,size,3):
            value = price_list[i]
            value = value[1:-1]
            self.price_list.append(value)

     #           self.daily_change_list = []
     #     for j in range(1,size,3):
     #               value = price_list[j]
     #             value = value[1:-1]
     #      self.daily_change_list.append(value)
     #    newSize = int(len(self.daily_change_list))
     #       self.daily_change_list = self.daily_change_list[0:newSize]
     #      print(newSize)
     #       print(self.daily_change_list)

     #    self.weekly_change_list = [] for k in range(2,size,3):     value = price_list[j]     value = value[1:-1]
     #        self.weekly_change_list.append(value) newSize = int(len(self.weekly_change_list)) self.weekly_change_list =
     #    self.weekly_change_list[0:newSize] print(newSize) print(self.weekly_change_list)

    #def getWebInformationList(self,cardname):
    #def getWebPriceList(self):
     #   return self.price_list

class CardSummary:

    #def __init__(self):

    def setHeader(self):
        #self.currentrow = 0
        header_list = ['Card name','Color','Rarity','Foil','Special','Number','Location','Price']
        self.open_workbook = openpyxl.load_workbook('final_magic.xlsx')
        self.currentsheet = self.open_workbook.active
        # if first sheet is included in new workbook, write there
        c = 1
        # item is an actual value of the element inside the list and NOT a pointercol =  - lesson learned from how loop work for lists
        for item in header_list:
            self.currentsheet.cell(row=1,column=c).value = item
            c += 1
        self.open_workbook.save('final_magic.xlsx')

    # Writes each merged combined line from card collection and web page to output file
    # Saves output file
    def writeSummaryRow(self):
    	# rowindex+1  , culomn =0 ?
    	# read from summary list from first item (loop through)
        self.currentrow += 1

    def saveCardInfo(self):  # transfer this to CardSummary class and adjust it

    	# create a new workbook-only once
        # if final_magic sheet does not exist create one:
        new_workbook = xlsxwriter.Workbook('final_magic.xlsx')
        # add one sheet and assign a name-only once
        new_sheet = new_workbook.add_worksheet('NewZen')

        i = 0

        # adding header -only once
        for item in cardInfo:
            xwrite = new_sheet.write(1,i,cardInfo[i])
            i += 1
        new_workbook.close()
        print('Card saved to new excel file')

###################################################################################################################

# Card Collection Test Suite
# implement the test suite with nose
oCC = CardCollection('/Users/mity/mypy/MTG_Collection_4_20_16.xlsx')
numSheets = oCC.getNumSheets()
numRows = oCC.getNumRows(46)
sheetName = oCC.getSheetName(46)
cardName = oCC.getCardName(46,1)
cardInfo = oCC.getAllCardInfo(46,1)

oCS = CardSummary()
output_file = oCS.setHeader()

#oWS = WebScraperMTG('http://www.mtggoldfish.com/index/ZEN#paper')
#priceList = oWS.getWebPriceList()

#print("Price List from object: ", priceList)

print('Total number of SHEETS in this file is:', numSheets)
print('Total number of ROWS in this sheet is:',numRows)
print('The SHEET NAME found is:',sheetName)
print('The CARD NAME is:',cardName)
print('The list of all required information is:', cardInfo)

# if numSheets == 67:
#     print("numSheets Check = TRUE")
# else:
#     ##print("numSheets Check = FALSE")
#     print("numsheet reported is FALSE, expected value is" , numSheets)

# if numRows == 323:
#     print("numRows Check = TRUE")
# else:
#     print("numRows Check = FALSE")

# if sheetName == "ZEN":
#     print("sheetName Check = TRUE")
# else:
#     print("sheetName Check = FALSE")

# if cardName == "Bala Ged Thief":
#     print("cardName Check = TRUE")
# else:
#     print("cardName Check = FALSE")

# if cardInfo[0] == 'Bala Ged Thief':
#     print("Cardname Check from list = TRUE")
# else:
#     print("Cardname Check from list = FALSE")

# if cardInfo[1] == 'B':
#      print("Color Check from list = TRUE")
# else:
#      print("Color Check from list = FALSE")

# if cardInfo[2] == 'Rare':
#     print("Rarity Check from list= TRUE")
# else:
#     print("Rarity Check from list=  FALSE")

# if cardInfo[3] == 'N':
#     print("Foil Check from list = TRUE")
# else:
#     print("Foil Check from list = FALSE")

# if cardInfo[4] == 'N':
#     print("Special Check from list = TRUE")
# else:
#     print("Special Check from list = FALSE")

# if cardInfo[5] == '1':
#     print("Number Check from list = TRUE")
# else:
#     print("Number from list = FALSE")

# if cardInfo[6] == 'Zendikar - Storage Box':
#     print("Location Check from list = TRUE")
# else:
#     print("Location Check from list = FALSE")
# cardName = oCC.getCardname(47,5)
# cardInfo = oCC.getAllCardInfo(47,5)

# print(cardName)
# if cardName == "Bloodchief Ascension":
#     print("cardname Check = TRUE")
# else:
#     print("cardname Check = FALSE")
<|endoftext|>


<commit_before>import commands
import os
import sys
import Queue
import settings
import threading
import traceback

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.normpath(os.path.join(BASE_PATH, '..'))
DEVICES_PATH = os.path.normpath(os.path.join(BASE_PATH, 'Hosts'))
THIRTYBIRDS_PATH = os.path.normpath(os.path.join(UPPER_PATH, 'thirtybirds_2_0'))

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from layer.live_looper import LiveLooper
from layer.looper_controller import LooperController

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def get_shelf_id(self):
        return self.hostname[11:][:1]

    def get_camera_id(self):
        return self.hostname[12:]

    def create_image_file_name(self, timestamp, light_level, process_type):
        return "{}_{}_{}_{}_{}.png".format(timestamp, self.get_shelf_id() ,  self.get_camera_id(), light_level, process_type)

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

class Layer(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.network.thirtybirds.subscribe_to_topic("layer_3_record")
        self.network.thirtybirds.subscribe_to_topic("layer_3_play")
        self.network.thirtybirds.subscribe_to_topic("layer_3_volume")
        self.network.thirtybirds.subscribe_to_topic("layer_speed")
        self.network.thirtybirds.subscribe_to_topic("clear_loop")
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("client_monitor_request")
        self.network.thirtybirds.subscribe_to_topic("mandala_device_request")
        self.status = {
            "avl-layer-1":"pass", # because this passes if it can respond.  maybe better tests in future
        }

    def loop_callback(self):
        # print "Sending layer trigger 3"
        self.network.thirtybirds.send("layer_3_trigger", "1")

    def update_device_status(self, devicename, status):
        if self.status[devicename] != status:
            self.status[devicename] = status
            msg = [devicename, status]
            self.network.thirtybirds.send("mandala_device_status", msg)

    def get_device_status(self):
        for devicename in self.status:
            msg = [devicename, self.status[devicename]]
            self.network.thirtybirds.send("mandala_device_status", msg)

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the thread of Layer
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the thread of Layer
        self.add_to_queue("__print__", topic_msg)
        # print "Layer.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        self.looper = LiveLooper()
        self.looper.setLoopCallback(self.loop_callback)
        self.looperController = LooperController(self.looper)
        self.looper.start()
        while True:
            try:
                topic, msg = self.queue.get(True)

                if topic == "mandala_device_request":
                    self.get_device_status()

                if topic == "layer_3_record":
                    if msg:
                        self.looperController.handleShortPedalDown()
                    else:
                        self.looperController.handleShortPedalUp()
                elif topic == "layer_3_play":
                    if msg:
                        self.looperController.handleLongPedalDown()
                    else:
                        self.looperController.handleLongPedalUp()
                elif topic == "layer_3_volume":
                    self.looperController.setVolume(msg)
                elif topic == "layer_speed":
                    self.looperController.setLoopLength(msg)
                elif topic == "clear_loop":
                    self.looperController.clear()
                elif topic == "__heartbeat__":
                    print "heartbeat received", msg
                elif topic == "__print__":
                    print "Layer.network_status_handler", msg
                else:
                    print ("Unhandled message type %s" % topic)

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    layer = Layer(hostname)
    layer.daemon = True
    layer.start()
    return layer
<commit_msg>adding mandala functions to more hosts<commit_after>import commands
import os
import sys
import Queue
import settings
import threading
import traceback

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.normpath(os.path.join(BASE_PATH, '..'))
DEVICES_PATH = os.path.normpath(os.path.join(BASE_PATH, 'Hosts'))
THIRTYBIRDS_PATH = os.path.normpath(os.path.join(UPPER_PATH, 'thirtybirds_2_0'))

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from layer.live_looper import LiveLooper
from layer.looper_controller import LooperController

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def get_shelf_id(self):
        return self.hostname[11:][:1]

    def get_camera_id(self):
        return self.hostname[12:]

    def create_image_file_name(self, timestamp, light_level, process_type):
        return "{}_{}_{}_{}_{}.png".format(timestamp, self.get_shelf_id() ,  self.get_camera_id(), light_level, process_type)

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

class Layer(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.network.thirtybirds.subscribe_to_topic("layer_3_record")
        self.network.thirtybirds.subscribe_to_topic("layer_3_play")
        self.network.thirtybirds.subscribe_to_topic("layer_3_volume")
        self.network.thirtybirds.subscribe_to_topic("layer_speed")
        self.network.thirtybirds.subscribe_to_topic("clear_loop")
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("client_monitor_request")
        self.network.thirtybirds.subscribe_to_topic("mandala_device_request")
        self.status = {
            "avl-layer-3":"pass", # because this passes if it can respond.  maybe better tests in future
        }

    def loop_callback(self):
        # print "Sending layer trigger 3"
        self.network.thirtybirds.send("layer_3_trigger", "1")

    def update_device_status(self, devicename, status):
        if self.status[devicename] != status:
            self.status[devicename] = status
            msg = [devicename, status]
            self.network.thirtybirds.send("mandala_device_status", msg)

    def get_device_status(self):
        for devicename in self.status:
            msg = [devicename, self.status[devicename]]
            self.network.thirtybirds.send("mandala_device_status", msg)

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the thread of Layer
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the thread of Layer
        self.add_to_queue("__print__", topic_msg)
        # print "Layer.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        self.looper = LiveLooper()
        self.looper.setLoopCallback(self.loop_callback)
        self.looperController = LooperController(self.looper)
        self.looper.start()
        while True:
            try:
                topic, msg = self.queue.get(True)

                if topic == "mandala_device_request":
                    self.get_device_status()

                if topic == "layer_3_record":
                    if msg:
                        self.looperController.handleShortPedalDown()
                    else:
                        self.looperController.handleShortPedalUp()
                elif topic == "layer_3_play":
                    if msg:
                        self.looperController.handleLongPedalDown()
                    else:
                        self.looperController.handleLongPedalUp()
                elif topic == "layer_3_volume":
                    self.looperController.setVolume(msg)
                elif topic == "layer_speed":
                    self.looperController.setLoopLength(msg)
                elif topic == "clear_loop":
                    self.looperController.clear()
                elif topic == "__heartbeat__":
                    print "heartbeat received", msg
                elif topic == "__print__":
                    print "Layer.network_status_handler", msg
                else:
                    print ("Unhandled message type %s" % topic)

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    layer = Layer(hostname)
    layer.daemon = True
    layer.start()
    return layer
<|endoftext|>


<commit_before>import os
import sys
import settings
import traceback
import threading
import Queue
import crystal_helpers as crystal

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
DEVICES_PATH = "%s/Hosts/" % (BASE_PATH )
THIRTYBIRDS_PATH = "%s/thirtybirds_2_0" % (UPPER_PATH )

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from thirtybirds_2_0.Network.manager import init as network_init

class GainRampThread(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.currentGains = [0x00, 0x00, 0x00]
        self.targetGains = [0x00, 0x00, 0x00]
        self.rampTimePerIncrement = 0.001
        self.queue = Queue.Queue()

    def setRampTime(self, r):
        self.rampTimePerIncrement = r;

    def setTargetGain(self, crystalIndex, gain):
        print "setTargetGain", crystalIndex, gain
        self.queue.put((crystalIndex, gain))

    def run(self):
        while True:
            while not self.queue.empty():
                crystalIndex, gain = self.queue.get()
                self.targetGains[crystalIndex] = gain & 0xFF
            for i in range(0, 3):
                if (self.targetGains[i] > self.currentGains[i]):
                    self.currentGains[i] = self.currentGains[i] + 1
                    print "GainRampThread", i, self.currentGains[i]
                    crystal.set_volume(i, self.currentGains[i])
                elif (self.targetGains[i] < self.currentGains[i]):
                    self.currentGains[i] = self.currentGains[i] - 1
                    print "GainRampThread", i, self.currentGains[i]
                    crystal.set_volume(i, self.currentGains[i])
            time.sleep(self.rampTimePerIncrement)

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.gainRampThread = GainRampThread()
        self.gainRampThread.daemon = True
        self.gainRampThread.start()
        #self.gainRampThread = gainRampThread
        # default intermediate frequency
        self.xtal_freq = 167465.0
        self.f_offset = 0           # adjust output freq
        # get voice messages
        self.network.thirtybirds.subscribe_to_topic("voice_1")

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        while True:
            try:
                topic, msg = self.queue.get(True)
                if topic == "voice_1":
                    params = []
                    # mute if volume is below threshold
                    thresh = [0.01, 0.1, 0.1]
                    for i in xrange(6):
                        param = 0 if msg[1] < thresh[0] else msg[i]                   # master
                        param = 0 if msg[3] < thresh[1] and i in (2,3) else param     # subvoice 1
                        param = 0 if msg[5] < thresh[2] and i in (4,5) else param     # subvoice 2
                        params.append(param)
                    freq_root, vol, freq_sub1, vol_sub1, freq_sub2, vol_sub2 = params
                    # update intermediate frequency if new data is available
                    self.xtal_freq = crystal.measure_xtal_freq() or self.xtal_freq
                    print params, self.xtal_freq
                    # subvoice 1 (fundamental) frequency and voice volume
                    crystal.set_freq(0, self.xtal_freq - (freq_root + self.f_offset))
                    self.gainRampThread.setTargetGain(0, map_master_volume(vol))
                    # subvoice 2 frequency and volume
                    crystal.set_freq(1, self.xtal_freq - (freq_sub1 + self.f_offset))
                    self.gainRampThread.setTargetGain(1, map_subvoice_volume(vol_sub1))
                    # subvoice 3 frequency and volume
                    crystal.set_freq(2, self.xtal_freq - (freq_sub2 + self.f_offset))
                    self.gainRampThread.setTargetGain(2, map_subvoice_volume(vol_sub2))

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    crystal.init()
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

def map_subvoice_volume(level):
    return map_volume(level, 180, 40)

def map_master_volume(level,):
    return map_volume(level, 150, 35)

def map_volume(level, min, scale):
    return 0 if level == 0 else int(min + level * scale)<commit_msg>working on volume value ramps<commit_after>import os
import sys
import settings
import traceback
import threading
import Queue
import crystal_helpers as crystal

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
DEVICES_PATH = "%s/Hosts/" % (BASE_PATH )
THIRTYBIRDS_PATH = "%s/thirtybirds_2_0" % (UPPER_PATH )

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from thirtybirds_2_0.Network.manager import init as network_init

class GainRampThread(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.currentGains = [0x00, 0x00, 0x00]
        self.targetGains = [0x00, 0x00, 0x00]
        self.rampTimePerIncrement = 0.001
        self.queue = Queue.Queue()

    def setRampTime(self, r):
        self.rampTimePerIncrement = r;

    def setTargetGain(self, crystalIndex, gain):
        print "setTargetGain", crystalIndex, gain
        self.queue.put((crystalIndex, gain))

    def run(self):
        while True:
            print "self.queue.empty()", self.queue.empty()
            while not self.queue.empty():
                crystalIndex, gain = self.queue.get()
                self.targetGains[crystalIndex] = gain & 0xFF
            for i in range(0, 3):
                if (self.targetGains[i] > self.currentGains[i]):
                    self.currentGains[i] = self.currentGains[i] + 1
                    print "GainRampThread", i, self.currentGains[i]
                    crystal.set_volume(i, self.currentGains[i])
                elif (self.targetGains[i] < self.currentGains[i]):
                    self.currentGains[i] = self.currentGains[i] - 1
                    print "GainRampThread", i, self.currentGains[i]
                    crystal.set_volume(i, self.currentGains[i])
            time.sleep(self.rampTimePerIncrement)

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.gainRampThread = GainRampThread()
        self.gainRampThread.daemon = True
        self.gainRampThread.start()
        #self.gainRampThread = gainRampThread
        # default intermediate frequency
        self.xtal_freq = 167465.0
        self.f_offset = 0           # adjust output freq
        # get voice messages
        self.network.thirtybirds.subscribe_to_topic("voice_1")

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        while True:
            try:
                topic, msg = self.queue.get(True)
                if topic == "voice_1":
                    params = []
                    # mute if volume is below threshold
                    thresh = [0.01, 0.1, 0.1]
                    for i in xrange(6):
                        param = 0 if msg[1] < thresh[0] else msg[i]                   # master
                        param = 0 if msg[3] < thresh[1] and i in (2,3) else param     # subvoice 1
                        param = 0 if msg[5] < thresh[2] and i in (4,5) else param     # subvoice 2
                        params.append(param)
                    freq_root, vol, freq_sub1, vol_sub1, freq_sub2, vol_sub2 = params
                    # update intermediate frequency if new data is available
                    self.xtal_freq = crystal.measure_xtal_freq() or self.xtal_freq
                    print params, self.xtal_freq
                    # subvoice 1 (fundamental) frequency and voice volume
                    crystal.set_freq(0, self.xtal_freq - (freq_root + self.f_offset))
                    self.gainRampThread.setTargetGain(0, map_master_volume(vol))
                    # subvoice 2 frequency and volume
                    crystal.set_freq(1, self.xtal_freq - (freq_sub1 + self.f_offset))
                    self.gainRampThread.setTargetGain(1, map_subvoice_volume(vol_sub1))
                    # subvoice 3 frequency and volume
                    crystal.set_freq(2, self.xtal_freq - (freq_sub2 + self.f_offset))
                    self.gainRampThread.setTargetGain(2, map_subvoice_volume(vol_sub2))

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    crystal.init()
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

def map_subvoice_volume(level):
    return map_volume(level, 180, 40)

def map_master_volume(level,):
    return map_volume(level, 150, 35)

def map_volume(level, min, scale):
    return 0 if level == 0 else int(min + level * scale)<|endoftext|>


<commit_before>"""
pin info:
  crystal frequency counter is connected via RS232
    any USB interface
  digital potentiometer for harmonic volume is connected via I2C
    pin 1: +3.3VDC
    pin 3: I2C Serial Data
    pin 5: I2C Serial Clock
    pin 6: Ground
  AD9833 frequecy generator chip #1 is connected via SPI
    pin 21: SPI MISO
    pin 19: SPI MOSI
    pin 23: SPI Serial Clock
    pin 29: Chip Select 1
    pin 6: Ground
  AD9833 frequecy generator chip #2 is connected via SPI
    pin 21: SPI MISO
    pin 19: SPI MOSI
    pin 23: SPI Serial Clock
    pin 31: Chip Select 2
    pin 6: Ground
  AD9833 frequecy generator chip #3 is connected via SPI
    pin 21: SPI MISO
    pin 19: SPI MOSI
    pin 23: SPI Serial Clock
    pin 33: Chip Select 3
    pin 6: Ground
  master volume amplifier board is connected via SPI
    pin 16: SPI MOSI
    pin 18: SPI Serial Clock
"""

import commands
import os
import sys
import settings
import traceback
import threading
import Queue
import crystal_helpers as crystal

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
DEVICES_PATH = "%s/Hosts/" % (BASE_PATH )
THIRTYBIRDS_PATH = "%s/thirtybirds_2_0" % (UPPER_PATH )

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def get_shelf_id(self):
        return self.hostname[11:][:1]

    def get_camera_id(self):
        return self.hostname[12:]

    def create_image_file_name(self, timestamp, light_level, process_type):
        return "{}_{}_{}_{}_{}.png".format(timestamp, self.get_shelf_id() ,  self.get_camera_id(), light_level, process_type)

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.utils = Utils(hostname)

        # default intermediate frequency
        self.xtal_freq = 96131.8
        self.f_offset = 0           # adjust output freq

        # get voice messages
        self.network.thirtybirds.subscribe_to_topic("voice_3")
        self.network.thirtybirds.subscribe_to_topic("client_monitor_request")

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        while True:
            try:
                topic, msg = self.queue.get(True)
                print topic, msg
                if topic == "voice_3":

                    params = []
                    # mute if volume is below threshold
                    thresh = [0.01, 0.01, 0.01]
                    for i in xrange(6):
                        param = 0 if msg[1] < thresh[0] else msg[i]                   # master
                        param = 0 if msg[3] < thresh[1] and i in (2,3) else param     # subvoice 1
                        param = 0 if msg[5] < thresh[2] and i in (4,5) else param     # subvoice 2
                        params.append(param)
                    freq_root, vol, freq_sub1, vol_sub1, freq_sub2, vol_sub2 = params
                    vol = max(vol-0.13, 0)
                    vol_sub1 = max(vol_sub1-0.09, 0)
                    vol_sub2 = max(vol_sub2-0.09, 0)
                    print "volume=", vol, vol_sub1, vol_sub2
                    # update intermediate frequency if new data is available
                    #measure_xtal_freq = crystal.measure_xtal_freq()
                    self.xtal_freq = crystal.measure_xtal_freq() or self.xtal_freq
                    print "adjusted:", params, self.xtal_freq

                    # subvoice 1 (fundamental) frequency
                    crystal.set_freq(0, vol and (self.xtal_freq - (freq_root + self.f_offset)))

                    # subvoice 2 frequency and volume
                    crystal.set_freq(1, vol_sub1 and (self.xtal_freq - (freq_sub1 + self.f_offset)))
                    crystal.set_volume(1, map_subvoice_volume(vol_sub1))

                    # subvoice 3 frequency and volume
                    crystal.set_freq(2, vol_sub2 and (self.xtal_freq - (freq_sub2 + self.f_offset)))
                    crystal.set_volume(2, map_subvoice_volume(vol_sub2))

                if topic == "client_monitor_request":
                    self.network.thirtybirds.send("client_monitor_response", self.utils.get_client_status())

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

mainloop = False

def init(hostname):
    global mainloop
    crystal.init()

    mainloop= main = Main(hostname)
    main.daemon = True
    main.start()
    return main

def map_subvoice_volume(level):
    # return map_volume(level, 0, 100)
    return map_volume(level, 0, 200)
    # return map_volume(level, 150, 100)

def map_volume(level, min, scale):
    return 0 if level == 0 else int(min + level * scale)<commit_msg>adding mandala tests to voice-3<commit_after>"""
pin info:
  crystal frequency counter is connected via RS232
    any USB interface
  digital potentiometer for harmonic volume is connected via I2C
    pin 1: +3.3VDC
    pin 3: I2C Serial Data
    pin 5: I2C Serial Clock
    pin 6: Ground
  AD9833 frequecy generator chip #1 is connected via SPI
    pin 21: SPI MISO
    pin 19: SPI MOSI
    pin 23: SPI Serial Clock
    pin 29: Chip Select 1
    pin 6: Ground
  AD9833 frequecy generator chip #2 is connected via SPI
    pin 21: SPI MISO
    pin 19: SPI MOSI
    pin 23: SPI Serial Clock
    pin 31: Chip Select 2
    pin 6: Ground
  AD9833 frequecy generator chip #3 is connected via SPI
    pin 21: SPI MISO
    pin 19: SPI MOSI
    pin 23: SPI Serial Clock
    pin 33: Chip Select 3
    pin 6: Ground
  master volume amplifier board is connected via SPI
    pin 16: SPI MOSI
    pin 18: SPI Serial Clock
"""

import commands
import crystal_helpers as crystal
import os
import Queue
import sys
import settings
import time
import traceback
import threading

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
DEVICES_PATH = "%s/Hosts/" % (BASE_PATH )
THIRTYBIRDS_PATH = "%s/thirtybirds_2_0" % (UPPER_PATH )

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def get_shelf_id(self):
        return self.hostname[11:][:1]

    def get_camera_id(self):
        return self.hostname[12:]

    def create_image_file_name(self, timestamp, light_level, process_type):
        return "{}_{}_{}_{}_{}.png".format(timestamp, self.get_shelf_id() ,  self.get_camera_id(), light_level, process_type)

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.utils = Utils(hostname)
        #self.gainRampThread = gainRampThread
        # default intermediate frequency
        #self.xtal_freq = 167233.6
        self.xtal_freq = 96131.8
        self.f_offset = 0           # adjust output freq
        self.status = {
            "avl-voice-3":"pass", # because this passes if it can respond.  maybe better tests in future
            "avl-voice-3-crystal-frequency-counter":"unset",
            "avl-voice-3-harmonic-generators":"unset",
            "avl-voice-3-harmonic-volume":"unset",
        }
        # get voice messages
        self.network.thirtybirds.subscribe_to_topic("voice_3")
        self.network.thirtybirds.subscribe_to_topic("client_monitor_request")
        self.network.thirtybirds.subscribe_to_topic("mandala_device_request")

    def update_device_status(self, devicename, status):
        print "update_device_status",devicename, status
        if self.status[devicename] != status:
            self.status[devicename] = status
            msg = [devicename, status]
            self.network.thirtybirds.send("mandala_device_status", msg)

    def get_device_status(self, devicename, status):
        for devicename in self.status:
            msg = [devicename, self.status[devicename]]
            self.network.thirtybirds.send("mandala_device_status", msg)

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        while True:
            try:
                topic, msg = self.queue.get(True)
                print topic, msg
                if topic == "voice_3":

                    params = []
                    # mute if volume is below threshold
                    thresh = [0.01, 0.01, 0.01]
                    for i in xrange(6):
                        param = 0 if msg[1] < thresh[0] else msg[i]                   # master
                        param = 0 if msg[3] < thresh[1] and i in (2,3) else param     # subvoice 1
                        param = 0 if msg[5] < thresh[2] and i in (4,5) else param     # subvoice 2
                        params.append(param)
                    freq_root, vol, freq_sub1, vol_sub1, freq_sub2, vol_sub2 = params

                    #`vol = max(vol-0.03, 0)

                    try:
                        self.xtal_freq = crystal.measure_xtal_freq() or self.xtal_freq
                        self.update_device_status("avl-voice-3-crystal-frequency-counter", "pass")
                    except Exception as e:
                        self.update_device_status("avl-voice-3-crystal-frequency-counter", "fail")

                    try:
                        # subvoice 1 (fundamental) frequency
                        crystal.set_freq(0, vol and (self.xtal_freq - (freq_root + self.f_offset)))
                        # subvoice 2 frequency
                        crystal.set_freq(1, vol_sub1 and (self.xtal_freq - (freq_sub1 + self.f_offset)))
                        # subvoice 3 frequency
                        crystal.set_freq(2, vol_sub2 and (self.xtal_freq - (freq_sub2 + self.f_offset)))
                        self.update_device_status("avl-voice-3-harmonic-generators", "pass")
                    except Exception as e:
                        self.update_device_status("avl-voice-3-harmonic-generators", "fail")

                    try:
                        # subvoice 2 volume
                        crystal.set_volume(1, map_subvoice_volume(vol_sub1))
                        # subvoice 3 volume
                        crystal.set_volume(2, map_subvoice_volume(vol_sub2))
                        self.update_device_status("avl-voice-3-harmonic-volume", "pass")
                    except Exception as e:
                        self.update_device_status("avl-voice-3-harmonic-volume", "fail")

                if topic == "client_monitor_request":
                    self.network.thirtybirds.send("client_monitor_response", self.utils.get_client_status())

                if topic == "mandala_device_request":
                    self.get_device_status()

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    crystal.init()
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

def map_subvoice_volume(level):
    # return map_volume(level, 0, 100)
    return map_volume(level, 0, 200)
    # return map_volume(level, 150, 100)

def map_volume(level, min, scale):
    return 0 if level == 0 else int(min + level * scale)<|endoftext|>


<commit_before>from django.core.management import BaseCommand
from seedsource_core.django.seedsource.models import SeedZone
import subprocess
import os

class Command(BaseCommand):
    help = 'Facilitates converting of vector data into vector tiles.'

    def _create_folder(self, directory):
        try:
            if not os.path.exists(directory):
                os.makedirs(directory)
        except OSError:
            print('Error: Creating directory. ' + directory)

    def _write_out_yellow(self, output):
        self.stdout.write('\033[0;33m' + output + '\033[0m')

    def handle(self, *args, **options):
        self._create_folder("seedzones")

        for sz in SeedZone.objects.all():
            FormattedName = sz.name.replace(" ", "_").replace("(", "[").replace(")", "]").replace("/", "-")

            self._write_out_yellow("loading " + FormattedName)
            with open("geojson", "w") as f:
                f.write(sz.polygon.json)

            self._write_out_yellow("processing..")
            process = subprocess.Popen(''.join([
                "tippecanoe -o seedzones/",
                FormattedName,
                ".mbtiles -f -zg --drop-densest-as-needed geojson"
                ]), shell=True)
            ExitCode = process.wait()
            if ExitCode == 0:
                self.stdout.write(self.style.SUCCESS("Success\n"))
            else:
                self.stdout.write(self.style.ERROR("Error\n"))

        self._write_out_yellow("done")<commit_msg>improve error logging<commit_after>from django.core.management import BaseCommand
from seedsource_core.django.seedsource.models import SeedZone
import subprocess
import os

class Command(BaseCommand):
    help = 'Facilitates converting of vector data into vector tiles.'

    def _write_out(self, output):
        self.stdout.write('\033[0;33m' + output + '\033[0m')

    def handle(self, *args, **options):
        errors = []
        for sz in SeedZone.objects.all():
            formatted_name = sz.name\
                .replace(" ", "_")\
                .replace("(", "[")\
                .replace(")", "]")\
                .replace("/", "-")\
                .replace("&", "and")\
                .lower()

            self._write_out("loading " + formatted_name)
            with open("geojson", "w") as f:
                f.write(sz.polygon.json)

            self._write_out("processing..")
            process = subprocess.Popen(''.join([
                "cd .. && tippecanoe -o tiles/seedzones/",
                formatted_name,
                ".mbtiles -f -zg --drop-densest-as-needed source/geojson"
                ]), shell=True)
            exit_code = process.wait()
            if exit_code == 0:
                self.stdout.write(self.style.SUCCESS("Success\n"))
            else:
                errors.append(formatted_name)
                self.stdout.write(self.style.ERROR("Error\n"))

        self._write_out("Done\n")
        if errors:
            self._write_out("There were errors with the following:\n")
            self.stdout.write(self.style.ERROR("\n".join(errors)))<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Base class for State objects."""
import inspect
import os
from datetime import datetime
from typing import List

from .. import DB, _events, LOG
from ..utils.datetime_utils import datetime_from_isoformat
from ._keys import STATES_KEY

class StateObject:
    """Base class for state objects (service state & sdp state)."""

    def __init__(self, object_id: str, allowed_states: List[str],
                 allowed_transitions: dict, allowed_target_states: dict):
        """Initialise a state object.

        Args:
            allowed_states (List[str]): List of allowed states.
            allowed_transitions (dict): Dict of allowed state transitions
            allowed_target_states (dict): Dict of allowed target states

        """
        self._id = object_id
        self._type = STATES_KEY  # object type
        self._key = '{}:{}'.format(STATES_KEY, self._id)
        self._allowed_states = [state.lower() for state in allowed_states]
        self._allowed_transitions = self._dict_lower(allowed_transitions)
        self._allowed_target_states = self._dict_lower(allowed_target_states)
        if not DB.key_exists(self._key):
            DB.set_hash_values(self._key, self._initialise())

    @property
    def id(self) -> str:
        """Return the object id."""
        return self._id

    @property
    def allowed_states(self) -> List[str]:
        """Get list of allowed object states."""
        return self._allowed_states

    @property
    def allowed_state_transitions(self) -> dict:
        """Get dictionary of allowed state transitions."""
        return self._allowed_transitions

    @property
    def allowed_target_states(self) -> dict:
        """Get dictionary of allowed target states / commands."""
        return self._allowed_target_states

    @property
    def current_state(self) -> str:
        """Get the current state."""
        return DB.get_hash_value(self._key, 'current_state')

    @current_state.setter
    def current_state(self, value):
        """Set the current state."""
        self.update_current_state(value)

    @property
    def target_state(self) -> str:
        """Get the target state."""
        return DB.get_hash_value(self._key, 'target_state')

    @target_state.setter
    def target_state(self, value):
        """Set the target state."""
        self.update_target_state(value)

    @property
    def current_timestamp(self) -> datetime:
        """Get the current state timestamp."""
        timestamp = DB.get_hash_value(self._key, 'current_timestamp')
        return datetime_from_isoformat(timestamp)

    @property
    def target_timestamp(self) -> datetime:
        """Get the target state timestamp."""
        timestamp = DB.get_hash_value(self._key, 'target_timestamp')
        return datetime_from_isoformat(timestamp)

    def update_target_state(self, value: str) -> datetime:
        """Set the target state.

        Args:
            value (str): New value for target state

        Returns:
            datetime, update timestamp

        Raises:
            RuntimeError, if it is not possible to currently set the target
            state.
            ValueError, if the specified target stat is not allowed.

        """
        value = value.lower()
        current_state = self.current_state
        if current_state == 'unknown':
            raise RuntimeError("Unable to set target state when current state "
                               "is 'unknown'")

        allowed_target_states = self._allowed_transitions[current_state]

        LOG.debug('Updating target state of %s to %s', self._id, value)

        if value not in allowed_target_states:
            raise ValueError("Invalid target state: '{}'. {} can be "
                             "commanded to states: {}".
                             format(value, current_state,
                                    allowed_target_states))

        return self._update_state('target', value)

    def update_current_state(self, value: str) -> datetime:
        """Update the current state.

        Args:
            value (str): New value for sdp state

        Returns:
            datetime, update timestamp

        Raises:
            ValueError: If the specified current state is not allowed.

        """
        value = value.lower()
        current_state = self.current_state
        # IF the current state is unknown, it can be set to any of the allowed
        # states, otherwise only allow certain transitions.
        if current_state == 'unknown':
            allowed_transitions = self._allowed_states
        else:
            allowed_transitions = self._allowed_transitions[current_state]
            allowed_transitions.append(current_state)

        LOG.debug('Updating current state of %s to %s', self._id, value)

        if value not in allowed_transitions:
            raise ValueError("Invalid current state update: '{}'. '{}' can be "
                             "transitioned to states: {}"
                             .format(value, current_state,
                                     allowed_transitions))

        return self._update_state('current', value)

    ###########################################################################
    # Pub/Sub functions
    ###########################################################################

    @staticmethod
    def subscribe(subscriber: str) -> _events.EventQueue:
        """Subscribe to state events.

        Args:
            subscriber (str): Subscriber name.

        Returns:
            events.EventQueue, Event queue object for querying events.

        """
        return _events.subscribe(STATES_KEY, subscriber)

    @staticmethod
    def get_subscribers() -> List[str]:
        """Get the list of subscribers to state events.

        Returns:
            List[str], list of subscriber names.

        """
        return _events.get_subscribers(STATES_KEY)

    def publish(self, event_type: str, event_data: dict = None):
        """Publish an state event.

        Args:
            event_type (str): Type of event.
            event_data (dict, optional): Event data.

        """
        _stack = inspect.stack()
        _origin = (os.path.basename(_stack[3][1]) + '::' +
                   _stack[3][3]+'::L{}'.format(_stack[3][2]))
        _events.publish(event_type=event_type,
                        event_data=event_data,
                        object_type=self._type,
                        object_id=self._id,
                        object_key=self._key,
                        origin=_origin)

    ###########################################################################
    # Private functions
    ###########################################################################

    def _initialise(self, initial_state: str = 'unknown') -> dict:
        """Return a dictionary used to initialise a state object.

        This method is used to obtain a dictionary/hash describing the initial
        state of SDP or a service in SDP.

        Args:
            initial_state (str): Initial state.

        Returns:
            dict, Initial state configuration

        """
        initial_state = initial_state.lower()
        if initial_state != 'unknown' and \
                initial_state not in self._allowed_states:
            raise ValueError('Invalid initial state: {}'.format(initial_state))
        _initial_state = dict(
            current_state=initial_state,
            target_state=initial_state,
            current_timestamp=datetime.utcnow().isoformat(),
            target_timestamp=datetime.utcnow().isoformat())
        return _initial_state

    def _update_state(self, state_type: str, value: str) -> datetime:
        """Update the state of type specified (current or target).

        Args:
            state_type(str): Type of state to update, current or target.
            value (str): New state value.

        Returns:
            timestamp, current time

        """
        timestamp = datetime.utcnow()
        field = '{}_state'.format(state_type)
        old_state = DB.get_hash_value(self._key, field)
        DB.set_hash_value(self._key, field, value, pipeline=True)
        DB.set_hash_value(self._key, '{}_timestamp'.format(state_type),
                          timestamp.isoformat(), pipeline=True)
        DB.execute()

        # Publish an event to notify subscribers of the change in state
        self.publish('{}_state_updated'.format(state_type),
                     event_data=dict(state=value, old_state=old_state))

        return timestamp

    @staticmethod
    def _dict_lower(dictionary: dict):
        """Convert allowed state transitions / target states to lowercase."""
        return {key.lower(): [value.lower() for value in value]
                for key, value in dictionary.items()}

    def __repr__(self):
        """Get a string representation of a State object."""
        return self._id
<commit_msg>added state object method to get the event queue for a given subscriber.<commit_after># -*- coding: utf-8 -*-
"""Base class for State objects."""
import inspect
import os
from datetime import datetime
from typing import List

from .. import DB, _events, LOG
from ..utils.datetime_utils import datetime_from_isoformat
from ._keys import STATES_KEY

class StateObject:
    """Base class for state objects (service state & sdp state)."""

    def __init__(self, object_id: str, allowed_states: List[str],
                 allowed_transitions: dict, allowed_target_states: dict):
        """Initialise a state object.

        Args:
            allowed_states (List[str]): List of allowed states.
            allowed_transitions (dict): Dict of allowed state transitions
            allowed_target_states (dict): Dict of allowed target states

        """
        self._id = object_id
        self._type = STATES_KEY  # object type
        self._key = '{}:{}'.format(STATES_KEY, self._id)
        self._allowed_states = [state.lower() for state in allowed_states]
        self._allowed_transitions = self._dict_lower(allowed_transitions)
        self._allowed_target_states = self._dict_lower(allowed_target_states)
        if not DB.key_exists(self._key):
            DB.set_hash_values(self._key, self._initialise())

    @property
    def id(self) -> str:
        """Return the object id."""
        return self._id

    @property
    def allowed_states(self) -> List[str]:
        """Get list of allowed object states."""
        return self._allowed_states

    @property
    def allowed_state_transitions(self) -> dict:
        """Get dictionary of allowed state transitions."""
        return self._allowed_transitions

    @property
    def allowed_target_states(self) -> dict:
        """Get dictionary of allowed target states / commands."""
        return self._allowed_target_states

    @property
    def current_state(self) -> str:
        """Get the current state."""
        return DB.get_hash_value(self._key, 'current_state')

    @current_state.setter
    def current_state(self, value):
        """Set the current state."""
        self.update_current_state(value)

    @property
    def target_state(self) -> str:
        """Get the target state."""
        return DB.get_hash_value(self._key, 'target_state')

    @target_state.setter
    def target_state(self, value):
        """Set the target state."""
        self.update_target_state(value)

    @property
    def current_timestamp(self) -> datetime:
        """Get the current state timestamp."""
        timestamp = DB.get_hash_value(self._key, 'current_timestamp')
        return datetime_from_isoformat(timestamp)

    @property
    def target_timestamp(self) -> datetime:
        """Get the target state timestamp."""
        timestamp = DB.get_hash_value(self._key, 'target_timestamp')
        return datetime_from_isoformat(timestamp)

    def update_target_state(self, value: str) -> datetime:
        """Set the target state.

        Args:
            value (str): New value for target state

        Returns:
            datetime, update timestamp

        Raises:
            RuntimeError, if it is not possible to currently set the target
            state.
            ValueError, if the specified target stat is not allowed.

        """
        value = value.lower()
        current_state = self.current_state
        if current_state == 'unknown':
            raise RuntimeError("Unable to set target state when current state "
                               "is 'unknown'")

        allowed_target_states = self._allowed_transitions[current_state]

        LOG.debug('Updating target state of %s to %s', self._id, value)

        if value not in allowed_target_states:
            raise ValueError("Invalid target state: '{}'. {} can be "
                             "commanded to states: {}".
                             format(value, current_state,
                                    allowed_target_states))

        return self._update_state('target', value)

    def update_current_state(self, value: str) -> datetime:
        """Update the current state.

        Args:
            value (str): New value for sdp state

        Returns:
            datetime, update timestamp

        Raises:
            ValueError: If the specified current state is not allowed.

        """
        value = value.lower()
        current_state = self.current_state
        # IF the current state is unknown, it can be set to any of the allowed
        # states, otherwise only allow certain transitions.
        if current_state == 'unknown':
            allowed_transitions = self._allowed_states
        else:
            allowed_transitions = self._allowed_transitions[current_state]
            allowed_transitions.append(current_state)

        LOG.debug('Updating current state of %s to %s', self._id, value)

        if value not in allowed_transitions:
            raise ValueError("Invalid current state update: '{}'. '{}' can be "
                             "transitioned to states: {}"
                             .format(value, current_state,
                                     allowed_transitions))

        return self._update_state('current', value)

    ###########################################################################
    # Pub/Sub functions
    ###########################################################################

    @staticmethod
    def subscribe(subscriber: str) -> _events.EventQueue:
        """Subscribe to state events.

        Args:
            subscriber (str): Subscriber name.

        Returns:
            events.EventQueue, Event queue object for querying events.

        """
        return _events.subscribe(STATES_KEY, subscriber)

    @staticmethod
    def get_subscribers() -> List[str]:
        """Get the list of subscribers to state events.

        Returns:
            List[str], list of subscriber names.

        """
        return _events.get_subscribers(STATES_KEY)

    def publish(self, event_type: str, event_data: dict = None):
        """Publish an state event.

        Args:
            event_type (str): Type of event.
            event_data (dict, optional): Event data.

        """
        _stack = inspect.stack()
        _origin = (os.path.basename(_stack[3][1]) + '::' +
                   _stack[3][3]+'::L{}'.format(_stack[3][2]))
        _events.publish(event_type=event_type,
                        event_data=event_data,
                        object_type=self._type,
                        object_id=self._id,
                        object_key=self._key,
                        origin=_origin)

    def get_event_queue(self, subscriber: str):
        """Get an event queue for the specified subscriber."""
        return _events.EventQueue(self._type, subscriber)

    ###########################################################################
    # Private functions
    ###########################################################################

    def _initialise(self, initial_state: str = 'unknown') -> dict:
        """Return a dictionary used to initialise a state object.

        This method is used to obtain a dictionary/hash describing the initial
        state of SDP or a service in SDP.

        Args:
            initial_state (str): Initial state.

        Returns:
            dict, Initial state configuration

        """
        initial_state = initial_state.lower()
        if initial_state != 'unknown' and \
                initial_state not in self._allowed_states:
            raise ValueError('Invalid initial state: {}'.format(initial_state))
        _initial_state = dict(
            current_state=initial_state,
            target_state=initial_state,
            current_timestamp=datetime.utcnow().isoformat(),
            target_timestamp=datetime.utcnow().isoformat())
        return _initial_state

    def _update_state(self, state_type: str, value: str) -> datetime:
        """Update the state of type specified (current or target).

        Args:
            state_type(str): Type of state to update, current or target.
            value (str): New state value.

        Returns:
            timestamp, current time

        """
        timestamp = datetime.utcnow()
        field = '{}_state'.format(state_type)
        old_state = DB.get_hash_value(self._key, field)
        DB.set_hash_value(self._key, field, value, pipeline=True)
        DB.set_hash_value(self._key, '{}_timestamp'.format(state_type),
                          timestamp.isoformat(), pipeline=True)
        DB.execute()

        # Publish an event to notify subscribers of the change in state
        self.publish('{}_state_updated'.format(state_type),
                     event_data=dict(state=value, old_state=old_state))

        return timestamp

    @staticmethod
    def _dict_lower(dictionary: dict):
        """Convert allowed state transitions / target states to lowercase."""
        return {key.lower(): [value.lower() for value in value]
                for key, value in dictionary.items()}

    def __repr__(self):
        """Get a string representation of a State object."""
        return self._id
<|endoftext|>