<commit_before>#
# quickSim.py
#
# A module that contains simulators that quickly produce a fund history.
#
#

# Python imports
import os
import cPickle
import sys
import datetime as dt

# 3rd Party Imports
import pandas as pand

# QSTK imports
from qstkutil import dateutil as du
from qstkutil import DataAccess as da

def quickSim( alloc, historic, start_cash, historicClose=None ):
    """
    @summary Quickly back tests an allocation for certain historical data,
             using a starting fund value
    @param alloc: DataMatrix containing timestamps to test as indices and
                 Symbols to test as columns, with _CASH symbol as the last
                 column
    @param historic: Historic dataframe of equity prices
    @param start_cash: integer specifing initial fund value
    @param historicClose: Optional, if provided we will buy using the closing
                          prices of the next day
    @return funds: TimeSeries with fund values for each day in the back test
    @rtype TimeSeries
    """

    #original quick simulator
    #not designed to handle shorts

    #check each row in alloc
    for row in range(0, len(alloc.values[:,0])):
        if(abs(alloc.values[row, :].sum()-1)>.0001):
            #print alloc.values[row, :]
            #print alloc.values[row, :].sum()
            print "warning, alloc row " + str(row) + \
            "does not sum to one, rebalancing"
            #if no allocation, all in cash
            if(alloc.values[row, :].sum()==0):
                alloc.values[row, -1]=1
            else:
                alloc.values[row, :] = alloc.values[row, :]  \
                / alloc.values[row, :].sum()

    #fix invalid days
    historic = historic.fillna(method='backfill')

    #add cash column
    #print historic.columns
    historic['_CASH'] = 1

    #print historic.columns

    closest = historic[historic.index <= alloc.index[0]]
    fund_ts = pand.Series( [start_cash], index = [closest.index[-1]] )
    shares = alloc.values[0, :] * fund_ts.values[-1] / closest.values[-1, :]
    cash_values = pand.DataMatrix( [shares*closest.values[-1, :]],
                              index=[closest.index[-1]] )

    #compute all trade
    for i in range(1, len(alloc.values[:, 0])):

        #get closest date(previous date)
        closest = historic[ historic.index <= alloc.index[i] ]

        #for loop to calculate fund daily (without rebalancing)
        for date in closest[ closest.index > fund_ts.index[-1] ].index:
            #compute and record total fund value (Sum(closest close * stocks))
            fund_ts = fund_ts.append( pand.Series(
                        [ (closest.xs(date) * shares).sum() ], index=[date] ) )

            cash_values = cash_values.append(
                pand.DataMatrix( [shares * closest.xs(date)], index=[date] ) )

        #distribute fund in accordance with alloc
        shares = alloc.values[i, :] * \
                 fund_ts.values[-1] / closest.xs( closest.index[-1] )

    #compute fund value for rest of historic data with final share distribution
    for date in historic[ historic.index > alloc.index[-1] ].index:
        if date in closest.index:
            fund_ts = fund_ts.append(
                      pand.Series( [ (closest.xs(date) * shares).sum() ],
                                   index=[date] ) )

    #return fund record
    return fund_ts

def _compute_short(arr):
    tally=0
    for i in range(0, len(arr)-1):
        if arr[i]<0:
            tally=tally+arr[i]
    return abs(tally)

def _compute_long(arr):
    tally=0
    for i in range(0, len(arr)-1):
        if arr[i]>0:
            tally=tally+arr[i]
    return tally

def _compute_leverage(arr, fundval):
    if fundval==0:
        return 0
    return (_compute_long(arr) - _compute_short(arr))/fundval

def shortingQuickSim(alloc, historic, start_cash, leverage):
    #shortingQuickSim
    #
    #designed to handle shorts
    #keeps track of leverage keeping it within paramaterized value
    #
    #ignore alloc cash column
    del alloc['_CASH']
    #fix invalid days
    historic = historic.fillna(method='backfill')

    #compute first trade
    closest = historic[historic.index<=alloc.index[0]]
    fund_ts = pand.Series( [start_cash], index=[closest.index[-1]] )
    shares = alloc.values[0, :] * fund_ts.values[-1] / closest.values[-1, :]
    cash_values = pand.DataMatrix( [shares * closest.values[-1, :]],
                                   index=[closest.index[-1]] )

    #compute all trades
    for i in range(1, len(alloc.values[:, 0])):
        #check leverage
        this_leverage = _compute_leverage(alloc.values[0, :])
        if this_leverage>leverage:
            print 'Warning, leverage of ',this_leverage,\
                  ' reached, exceeds leverage limit of ', leverage, '\n'
        #get closest date(previous date)
        closest=historic[ historic.index <= alloc.index[i] ]
        #for loop to calculate fund daily (without rebalancing)
        for date in closest[ closest.index > fund_ts.index[-1] ].index:
            #compute and record total fund value (Sum(closest close * stocks))
            fund_ts = fund_ts.append( pand.Series(
                      [(closest.xs(date) * shares).sum()], index=[date]) )
            cash_values = cash_values.append( pand.DataMatrix(
                                   [shares*closest.xs(date)], index=[date]) )

        #distribute fund in accordance with alloc
        shares = alloc.values[i, :] * (
                 fund_ts.values[-1] / closest.xs(closest.index[-1]) )

    #compute fund value for rest of historic data with final share distribution
    for date in historic[ historic.index > alloc.index[-1] ].index:
        if date in closest.index:
            fund_ts = fund_ts.append(
                      pand.Series( [(closest.xs(date) * shares).sum()],
                                   index=[date]) )

    #return fund record
    return fund_ts

def alloc_backtest(alloc, start):
    """
    @summary: Back tests an allocation from a pickle file. Uses a starting
              portfolio value of start.
    @param alloc: Name of allocation pickle file. Pickle file contains a
                  DataMatrix with timestamps as indexes and stock symbols as
                  columns, with the last column being the _CASH symbol,
                  indicating how much
    of the allocation is in cash.
    @param start: integer specifying the starting value of the portfolio
    @return funds: List of fund values indicating the value of the portfolio
                   throughout the back test.
    @rtype timeSeries
    """

    #read in alloc table from command line arguements
    alloc_input_file=open(alloc, "r")
    alloc = cPickle.load(alloc_input_file)

    # Get the data from the data store
    dataobj = da.DataAccess('Norgate')
    historic = dataobj.get_data( list(alloc.index), list(alloc.columns[0:-1]),
                                 "close" )

    #backtest
    funds = quickSim(alloc, historic, int(start))

    return funds

def strat_backtest1(strat, start, end, num, diff, startval):
    """
    @summary: Back tests a strategy defined in a python script that takes
              in a start
    and end date along with a starting value a set number of times.
    @param strat: filename of python script strategy
    @param start: starting date in a datetime object
    @param end: ending date in a datetime object
    @param num: number of tests to perform
    @param diff: offset in days of the tests
    @param startval: starting value of fund during back tests
    @return fundsmatrix: Datamatrix of fund values returned from each test
    @rtype datamatrix
    """
    fundsmatrix = []
    startdates = du.getNextNNYSEdays(start, num * diff, dt.timedelta(hours=16))
    enddates = du.getNextNNYSEdays(end, num * diff, dt.timedelta(hours=16))
    for i in range(0, num):
        os.system( 'python ' + strat + ' ' + startdates[i].strftime("%m-%d-%Y")\
                    + ' ' + enddates[i].strftime("%m-%d-%Y") + \
                    ' temp_alloc.pkl')

        funds = alloc_backtest('temp_alloc.pkl', startval)
        fundsmatrix.append(funds)
    return fundsmatrix

def strat_backtest2(strat, start, end, diff, dur, startval):
    """
    @summary: Back tests a strategy defined in a python script that takes in a
             start and end date along with a starting value over a given
             period.
    @param strat: filename of python script strategy
    @param start: starting date in a datetime object
    @param end: ending date in a datetime object
    @param diff: offset in days of the tests
    @param dur: length of a test
    @param startval: starting value of fund during back tests
    @return fundsmatrix: Datamatrix of fund values returned from each test
    @rtype datamatrix
    """
    fundsmatrix = []
    startdates = du.getNYSEdays( start, end, dt.timedelta(hours=16) )
    for i in range(0, len(startdates), diff):
        if( i + dur>=len(startdates) ):
            enddate = startdates[-1]
        else:
            enddate = startdates[i+dur]
        os.system('python ' + strat + ' ' + startdates[i].strftime("%m-%d-%Y")\
                   + ' ' + enddate.strftime("%m-%d-%Y") + ' temp_alloc.pkl')
        funds = alloc_backtest('temp_alloc.pkl', startval)
        fundsmatrix.append(funds)
    return fundsmatrix

if __name__ == "__main__":
    #
    # CmdlnQuickSim
    #
    # A function which runs a quick sim on an allocation provided via command line,
    # along with a starting cash value
    #
    # Allocation Backtest:
    # python quickSim.py -a allocation_fiel start_value output_file
    # python quickSim.py -a 'alloc_file.pkl' 1000 'fund_output.pkl'
    #
    # Strategy backtest:
    # python quickSim.py -s strategy start end start_value output_file
    # python quickSim.py -s 'strategy.py' '2/2/2007' '2/2/2009' 1000 'fund_output.pkl'
    #
    # Robust backtest:
    # python quickSim.py -r strategy start end days_between duration start_value output
    # python quickSim.py -r 'strategy.py' '1-1-2004' '1-1-2007' 7 28 10000 'out.pkl'
    # Drew Bratcher
    #

    if(sys.argv[1] == '-a'):
        funds=alloc_backtest(sys.argv[2], sys.argv[3])
        output = open(sys.argv[4], "w")
        cPickle.dump(funds, output)
    elif(sys.argv[1] == '-s'):
        t = map(int,  sys.argv[3].split('-'))
        startday = dt.datetime(t[2], t[0], t[1])
        t = map(int,  sys.argv[4].split('-'))
        endday = dt.datetime(t[2], t[0], t[1])
        fundsmatrix=strat_backtest1(sys.argv[2], startday, endday, 1, 0,
                                    int(sys.argv[5]))
        output = open(sys.argv[6], "w")
        cPickle.dump(fundsmatrix, output)
    elif(sys.argv[1] == '-r'):
        t = map(int, sys.argv[3].split('-'))
        startday = dt.datetime(t[2], t[0], t[1])
        t = map(int, sys.argv[4].split('-'))
        endday = dt.datetime(t[2], t[0], t[1])
        fundsmatrix=strat_backtest2(sys.argv[2], startday, endday,
                                    int(sys.argv[5]), int(sys.argv[6]),
                                    int(sys.argv[7]))
        output = open(sys.argv[8], "w")
        cPickle.dump(fundsmatrix, output)
    else:
        print 'invalid command line call'
        print 'use python quickSim.py -a alloc_pkl start_value output_pkl'
        print 'or python quickSim.py -s strategy start_date end_date' + \
              'start_value output_pkl'
        print 'or python quickSim.py -r strategy start_date end_date' + \
              ' test_offset_in_days duration start_value output_pkl'
<commit_msg>Formatting<commit_after>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on Mar 15, 2012

@author: Drew Bratcher, John Cornwell
@contact: JWCornV@gmail.com
@summary: Simulator which quickly produces a fund history.
'''

# Python imports
import os
import cPickle
import sys
import datetime as dt

# 3rd Party Imports
import pandas as pand

# QSTK imports
from qstkutil import dateutil as du
from qstkutil import DataAccess as da

def quickSim( alloc, historic, start_cash, historicClose=None ):
    """
    @summary Quickly back tests an allocation for certain historical data,
             using a starting fund value
    @param alloc: DataMatrix containing timestamps to test as indices and
                 Symbols to test as columns, with _CASH symbol as the last
                 column
    @param historic: Historic dataframe of equity prices
    @param start_cash: integer specifing initial fund value
    @param historicClose: Optional, if provided we will buy using the closing
                          prices of the next day
    @return funds: TimeSeries with fund values for each day in the back test
    @rtype TimeSeries
    """

    #original quick simulator
    #not designed to handle shorts

    #check each row in alloc
    for row in range(0, len(alloc.values[:, 0])):
        if(abs(alloc.values[row, :].sum()-1)>.0001):
            #print alloc.values[row, :]
            #print alloc.values[row, :].sum()
            print "warning, alloc row " + str(row) + \
            "does not sum to one, rebalancing"
            #if no allocation, all in cash
            if(alloc.values[row, :].sum()==0):
                alloc.values[row, -1] = 1
            else:
                alloc.values[row, :] = alloc.values[row, :]  \
                / alloc.values[row, :].sum()

    #fix invalid days
    historic = historic.fillna(method='backfill')

    #add cash column
    #print historic.columns
    historic['_CASH'] = 1

    #print historic.columns

    closest = historic[historic.index <= alloc.index[0]]
    fund_ts = pand.Series( [start_cash], index = [closest.index[-1]] )
    shares = alloc.values[0, :] * fund_ts.values[-1] / closest.values[-1, :]
    cash_values = pand.DataMatrix( [shares*closest.values[-1, :]],
                              index=[closest.index[-1]] )

    #compute all trade
    for i in range(1, len(alloc.values[:, 0])):

        #get closest date(previous date)
        closest = historic[ historic.index <= alloc.index[i] ]

        #for loop to calculate fund daily (without rebalancing)
        for date in closest[ closest.index > fund_ts.index[-1] ].index:
            #compute and record total fund value (Sum(closest close * stocks))
            fund_ts = fund_ts.append( pand.Series(
                        [ (closest.xs(date) * shares).sum() ], index=[date] ) )

            cash_values = cash_values.append(
                pand.DataMatrix( [shares * closest.xs(date)], index=[date] ) )

        #distribute fund in accordance with alloc
        shares = alloc.values[i, :] * \
                 fund_ts.values[-1] / closest.xs( closest.index[-1] )

    #compute fund value for rest of historic data with final share distribution
    for date in historic[ historic.index > alloc.index[-1] ].index:
        if date in closest.index:
            fund_ts = fund_ts.append(
                      pand.Series( [ (closest.xs(date) * shares).sum() ],
                                   index=[date] ) )

    #return fund record
    return fund_ts

def _compute_short(arr):
    ''' Computes total value of negative positions '''
    tally = 0
    for i in range(0, len(arr) - 1):
        if arr[i] < 0:
            tally = tally+arr[i]
    return abs(tally)

def _compute_long(arr):
    ''' Computes total value of positive positions '''
    tally = 0
    for i in range(0, len(arr) - 1):
        if arr[i] > 0:
            tally = tally+arr[i]
    return tally

def _compute_leverage(arr, fundval):
    ''' Computes percent leverage '''
    if fundval == 0:
        return 0
    return (_compute_long(arr) - _compute_short(arr))/fundval

def shortingQuickSim(alloc, historic, start_cash, leverage):
    ''' shortingQuickSim
        designed to handle shorts, keeps track of leverage keeping it within
        paramaterized value, ignore alloc cash column '''

    del alloc['_CASH']
    #fix invalid days
    historic = historic.fillna(method='backfill')

    #compute first trade
    closest = historic[historic.index <= alloc.index[0]]
    fund_ts = pand.Series( [start_cash], index = [closest.index[-1]] )
    shares = alloc.values[0, :] * fund_ts.values[-1] / closest.values[-1, :]
    cash_values = pand.DataMatrix( [shares * closest.values[-1, :]],
                                   index=[closest.index[-1]] )

    #compute all trades
    for i in range(1, len(alloc.values[:, 0])):
        #check leverage
        #TODO Find out what to use for fundvall below...
        this_leverage = _compute_leverage( alloc.values[0, :], start_cash )
        if this_leverage > leverage:
            print 'Warning, leverage of ', this_leverage, \
                  ' reached, exceeds leverage limit of ', leverage, '\n'
        #get closest date(previous date)
        closest = historic[ historic.index <= alloc.index[i] ]
        #for loop to calculate fund daily (without rebalancing)
        for date in closest[ closest.index > fund_ts.index[-1] ].index:
            #compute and record total fund value (Sum(closest close * stocks))
            fund_ts = fund_ts.append( pand.Series(
                      [(closest.xs(date) * shares).sum()], index=[date]) )
            cash_values = cash_values.append( pand.DataMatrix(
                                   [shares*closest.xs(date)], index=[date]) )

        #distribute fund in accordance with alloc
        shares = alloc.values[i, :] * (
                 fund_ts.values[-1] / closest.xs(closest.index[-1]) )

    #compute fund value for rest of historic data with final share distribution
    for date in historic[ historic.index > alloc.index[-1] ].index:
        if date in closest.index:
            fund_ts = fund_ts.append(
                      pand.Series( [(closest.xs(date) * shares).sum()],
                                   index=[date]) )

    #return fund record
    return fund_ts

def alloc_backtest(alloc, start):
    """
    @summary: Back tests an allocation from a pickle file. Uses a starting
              portfolio value of start.
    @param alloc: Name of allocation pickle file. Pickle file contains a
                  DataMatrix with timestamps as indexes and stock symbols as
                  columns, with the last column being the _CASH symbol,
                  indicating how much
    of the allocation is in cash.
    @param start: integer specifying the starting value of the portfolio
    @return funds: List of fund values indicating the value of the portfolio
                   throughout the back test.
    @rtype timeSeries
    """

    #read in alloc table from command line arguements
    alloc_input_file = open(alloc, "r")
    alloc = cPickle.load(alloc_input_file)

    # Get the data from the data store
    dataobj = da.DataAccess('Norgate')
    historic = dataobj.get_data( list(alloc.index), list(alloc.columns[0:-1]),
                                 "close" )

    #backtest
    funds = quickSim(alloc, historic, int(start))

    return funds

def strat_backtest1(strat, start, end, num, diff, startval):
    """
    @summary: Back tests a strategy defined in a python script that takes
              in a start
    and end date along with a starting value a set number of times.
    @param strat: filename of python script strategy
    @param start: starting date in a datetime object
    @param end: ending date in a datetime object
    @param num: number of tests to perform
    @param diff: offset in days of the tests
    @param startval: starting value of fund during back tests
    @return fundsmatrix: Datamatrix of fund values returned from each test
    @rtype datamatrix
    """
    fundsmatrix = []
    startdates = du.getNextNNYSEdays(start, num * diff, dt.timedelta(hours=16))
    enddates = du.getNextNNYSEdays(end, num * diff, dt.timedelta(hours=16))
    for i in range(0, num):
        os.system( 'python ' + strat + ' ' + startdates[i].strftime("%m-%d-%Y")\
                    + ' ' + enddates[i].strftime("%m-%d-%Y") + \
                    ' temp_alloc.pkl')

        funds = alloc_backtest('temp_alloc.pkl', startval)
        fundsmatrix.append(funds)
    return fundsmatrix

def strat_backtest2(strat, start, end, diff, dur, startval):
    """
    @summary: Back tests a strategy defined in a python script that takes in a
             start and end date along with a starting value over a given
             period.
    @param strat: filename of python script strategy
    @param start: starting date in a datetime object
    @param end: ending date in a datetime object
    @param diff: offset in days of the tests
    @param dur: length of a test
    @param startval: starting value of fund during back tests
    @return fundsmatrix: Datamatrix of fund values returned from each test
    @rtype datamatrix
    """
    fundsmatrix = []
    startdates = du.getNYSEdays( start, end, dt.timedelta(hours=16) )
    for i in range(0, len(startdates), diff):
        if( i + dur>=len(startdates) ):
            enddate = startdates[-1]
        else:
            enddate = startdates[i+dur]
        os.system('python ' + strat + ' ' + startdates[i].strftime("%m-%d-%Y")\
                   + ' ' + enddate.strftime("%m-%d-%Y") + ' temp_alloc.pkl')
        funds = alloc_backtest('temp_alloc.pkl', startval)
        fundsmatrix.append(funds)
    return fundsmatrix

def run_main():
    ''' Main program '''

    #
    # CmdlnQuickSim
    #
    # A function which runs a quick sim on an allocation provided via
    # command line,
    # along with a starting cash value
    #
    # Allocation Backtest:
    # python quickSim.py -a allocation_fiel start_value output_file
    # python quickSim.py -a 'alloc_file.pkl' 1000 'fund_output.pkl'
    #
    # Strategy backtest:
    # python quickSim.py -s strategy start end start_value output_file
    # python quickSim.py -s 'strategy.py' '2/2/2007' '2/2/2009' 1000
    # 'fund_output.pkl'
    #
    # Robust backtest:
    # python quickSim.py -r strategy start end days_between duration
    # start_value output
    # python quickSim.py -r 'strategy.py' '1-1-2004' '1-1-2007' 7 28 10000
    # 'out.pkl'
    # Drew Bratcher
    #

    if(sys.argv[1] == '-a'):
        funds = alloc_backtest(sys.argv[2], sys.argv[3])
        output = open(sys.argv[4], "w")
        cPickle.dump(funds, output)
    elif(sys.argv[1] == '-s'):
        t = map(int,  sys.argv[3].split('-'))
        startday = dt.datetime(t[2], t[0], t[1])
        t = map(int,  sys.argv[4].split('-'))
        endday = dt.datetime(t[2], t[0], t[1])
        fundsmatrix = strat_backtest1(sys.argv[2], startday, endday, 1, 0,
                                    int(sys.argv[5]))
        output = open(sys.argv[6], "w")
        cPickle.dump(fundsmatrix, output)
    elif(sys.argv[1] == '-r'):
        t = map(int, sys.argv[3].split('-'))
        startday = dt.datetime(t[2], t[0], t[1])
        t = map(int, sys.argv[4].split('-'))
        endday = dt.datetime(t[2], t[0], t[1])
        fundsmatrix = strat_backtest2(sys.argv[2], startday, endday,
                                    int(sys.argv[5]), int(sys.argv[6]),
                                    int(sys.argv[7]))
        output = open(sys.argv[8], "w")
        cPickle.dump(fundsmatrix, output)
    else:
        print 'invalid command line call'
        print 'use python quickSim.py -a alloc_pkl start_value output_pkl'
        print 'or python quickSim.py -s strategy start_date end_date' + \
              'start_value output_pkl'
        print 'or python quickSim.py -r strategy start_date end_date' + \
              ' test_offset_in_days duration start_value output_pkl'

if __name__ == "__main__":
    run_main()
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys, os

import mmap  # Thanks Steven @ http://stackoverflow.com/questions/4940032/search-for-string-in-txt-file-python

import subprocess

import readline

readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.parse_and_bind("set match-hidden-files off")

import re

from pychord import Chord
from pychord import ChordProgression

globalHalfTones = 0
songHalfTones = 0
applyCapoDropCorrection = True

def query(question, default):
    sys.stdout.write(question + " [" + default + "] ? ")
    choice = raw_input()
    if choice == '':
        return default
    return choice

def process( stringToProcess, processed):
    global songHalfTones
    #print 'String to process "' + stringToProcess + '".'
    afterSplit = re.split("  |-|!|\.\.\.|\.\.|: |\*|high|open|bass|riff|palm mute|notes|m6|madd11/|m7add11/|7sus2|8|m7b5|madd13|add13", stringToProcess, 1)  # 3rd parameter is maxsplit # Also works with single space, do this to catch faulty txt.
    #print '* Split by delimiters "' + str(afterSplit) + '".'
    #print 'songHalfTones:',songHalfTones
    if len(afterSplit[0]) != 0:
        chord = Chord(afterSplit[0])
        #print '* Extracted "' + chord.chord + '" chord.'
        chord.transpose( songHalfTones, "C#" )
        #print '* Transposed to "' + chord.chord + '" chord.'
        processed += chord.chord
        #print '* Processed after chord "' + processed + '".'
    #else:
        #print '* No chord to extract.'
    if len(afterSplit) == 1:
        return processed
    delimiterWas = ''
    if len(afterSplit[1]) == 0:
        delimiterWas = stringToProcess[len(afterSplit[0]):]
    else:
        delimiterWas = stringToProcess[len(afterSplit[0]):-len(afterSplit[1])]
    #print '* Delimiter was "' + delimiterWas + '".'
    processed += delimiterWas
    #print '* Processed after delimiter "' + processed + '".'
    #print '* Still must process "' + afterSplit[1] + '".'
    return process( afterSplit[1], processed )

def transpose(matchobj):
    global songHalfTones
    # debug
    print "--- " + matchobj.group(0)
    #exceptions:
    if matchobj.group(0).find("capo") != -1:
        if applyCapoDropCorrection:
            m = matchobj.group(0)
            got = re.findall('\d+', m)
            if len(got) != 1:
                print '*** ERROR (len(got) != 1)'
                quit()
            print '*** capo:',int(got[0])
            songHalfTones += int(got[0])
            print '*** new songHalfTones:',songHalfTones
        return matchobj.group(0)
    if matchobj.group(0).find("drop") != -1:
        if applyCapoDropCorrection:
            m = matchobj.group(0)
            got = re.findall('\d+', m)
            if len(got) != 1:
                print '*** ERROR (len(got) != 1)'
                quit()
            print '*** drop:',int(got[0])
            songHalfTones -= int(got[0])
            print '*** new songHalfTones:',songHalfTones
        return matchobj.group(0)
    if matchobj.group(0).find("bpm") != -1:
        return matchobj.group(0)
    if matchobj.group(0).find("(all") != -1:
        return matchobj.group(0)
    if matchobj.group(0).find("(mute)") != -1:
        return matchobj.group(0)
    #actual process:
    betweenParenthesis = matchobj.group(0).replace("(","").replace(")","")
    #print betweenParenthesis
    final = process( betweenParenthesis, "" )
    # debug
    print "+++ " + "(" + final + ")"
    return "(" + final + ")"

if __name__ == '__main__':

    print("-----------------------------------")
    print("Welcome to song-directory-transpose")
    print("-----------------------------------")

    # Query song directory path string
    songDirectory = query("Please specify the path of the song (input) directory","/opt/Dropbox/chords/0-GUITAR/english")
    print("Will use song directory (input): " + songDirectory)

    # Query transposed song directory path string
    transposedSongDirectory = query("Please specify the path of the transposed song (output) directory","tmp")

    if os.path.isdir(transposedSongDirectory):
        yesNo = query('Path "' + transposedSongDirectory + '" already exists, are you sure (confirm with "y" or "yes" without quotes)','yes')
        if yesNo != "yes" and yesNo != "y":
            print "Ok, bye!"
            quit()
        else:
            print("Will use (existing) transposed song directory (output): " + transposedSongDirectory)
    else:
        os.makedirs(transposedSongDirectory)
        print("Will use (newly created) transposed song directory (output): " + transposedSongDirectory)

    # Query transposition
    globalHalfTones = int( query("Please specify half tones of transposition (-5 for soprano ukelele and guitalele)","0") )
    print("Will use half tones of transposition: " + str(globalHalfTones))

    # Query transposition
    while True:
        yesNo = query('Apply capo/drop correction (confirm with "y" or "yes" without quotes)?','yes')
        if yesNo == "yes" or yesNo == "y":
            print("Will apply capo/drop correction")
            applyCapoDropCorrection = True
            break
        elif yesNo == "no" or yesNo == "n":
            print("Will not apply capo/drop correction")
            applyCapoDropCorrection = False
            break

    print("----------------------")

    for dirname, dirnames, filenames in os.walk( songDirectory ):
        for filename in sorted(filenames):
            songHalfTones = globalHalfTones
            #debug
            print filename
            print '*** songHalfTones:',songHalfTones
            name, extension = os.path.splitext(filename)
            songIn = open( os.path.join(dirname, filename) )
            songOut = open( os.path.join(transposedSongDirectory, filename), "w" )
            contents = songIn.read()
            contents = re.sub("\([^)]*\)", transpose, contents)
            songOut.write(contents)
            songOut.close()
            songIn.close()

<commit_msg>7 equivalent to -5 transposition, #14<commit_after>#!/usr/bin/python

import sys, os

import mmap  # Thanks Steven @ http://stackoverflow.com/questions/4940032/search-for-string-in-txt-file-python

import subprocess

import readline

readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.parse_and_bind("set match-hidden-files off")

import re

from pychord import Chord
from pychord import ChordProgression

globalHalfTones = 0
songHalfTones = 0
applyCapoDropCorrection = True

def query(question, default):
    sys.stdout.write(question + " [" + default + "] ? ")
    choice = raw_input()
    if choice == '':
        return default
    return choice

def process( stringToProcess, processed):
    global songHalfTones
    #print 'String to process "' + stringToProcess + '".'
    afterSplit = re.split("  |-|!|\.\.\.|\.\.|: |\*|high|open|bass|riff|palm mute|notes|m6|madd11/|m7add11/|7sus2|8|m7b5|madd13|add13", stringToProcess, 1)  # 3rd parameter is maxsplit # Also works with single space, do this to catch faulty txt.
    #print '* Split by delimiters "' + str(afterSplit) + '".'
    #print 'songHalfTones:',songHalfTones
    if len(afterSplit[0]) != 0:
        chord = Chord(afterSplit[0])
        #print '* Extracted "' + chord.chord + '" chord.'
        chord.transpose( songHalfTones, "C#" )
        #print '* Transposed to "' + chord.chord + '" chord.'
        processed += chord.chord
        #print '* Processed after chord "' + processed + '".'
    #else:
        #print '* No chord to extract.'
    if len(afterSplit) == 1:
        return processed
    delimiterWas = ''
    if len(afterSplit[1]) == 0:
        delimiterWas = stringToProcess[len(afterSplit[0]):]
    else:
        delimiterWas = stringToProcess[len(afterSplit[0]):-len(afterSplit[1])]
    #print '* Delimiter was "' + delimiterWas + '".'
    processed += delimiterWas
    #print '* Processed after delimiter "' + processed + '".'
    #print '* Still must process "' + afterSplit[1] + '".'
    return process( afterSplit[1], processed )

def transpose(matchobj):
    global songHalfTones
    # debug
    print "--- " + matchobj.group(0)
    #exceptions:
    if matchobj.group(0).find("capo") != -1:
        if applyCapoDropCorrection:
            m = matchobj.group(0)
            got = re.findall('\d+', m)
            if len(got) != 1:
                print '*** ERROR (len(got) != 1)'
                quit()
            print '*** capo:',int(got[0])
            songHalfTones += int(got[0])
            print '*** new songHalfTones:',songHalfTones
        return matchobj.group(0)
    if matchobj.group(0).find("drop") != -1:
        if applyCapoDropCorrection:
            m = matchobj.group(0)
            got = re.findall('\d+', m)
            if len(got) != 1:
                print '*** ERROR (len(got) != 1)'
                quit()
            print '*** drop:',int(got[0])
            songHalfTones -= int(got[0])
            print '*** new songHalfTones:',songHalfTones
        return matchobj.group(0)
    if matchobj.group(0).find("bpm") != -1:
        return matchobj.group(0)
    if matchobj.group(0).find("(all") != -1:
        return matchobj.group(0)
    if matchobj.group(0).find("(mute)") != -1:
        return matchobj.group(0)
    #actual process:
    betweenParenthesis = matchobj.group(0).replace("(","").replace(")","")
    #print betweenParenthesis
    final = process( betweenParenthesis, "" )
    # debug
    print "+++ " + "(" + final + ")"
    return "(" + final + ")"

if __name__ == '__main__':

    print("-----------------------------------")
    print("Welcome to song-directory-transpose")
    print("-----------------------------------")

    # Query song directory path string
    songDirectory = query("Please specify the path of the song (input) directory","/opt/Dropbox/chords/0-GUITAR/english")
    print("Will use song directory (input): " + songDirectory)

    # Query transposed song directory path string
    transposedSongDirectory = query("Please specify the path of the transposed song (output) directory","tmp")

    if os.path.isdir(transposedSongDirectory):
        yesNo = query('Path "' + transposedSongDirectory + '" already exists, are you sure (confirm with "y" or "yes" without quotes)','yes')
        if yesNo != "yes" and yesNo != "y":
            print "Ok, bye!"
            quit()
        else:
            print("Will use (existing) transposed song directory (output): " + transposedSongDirectory)
    else:
        os.makedirs(transposedSongDirectory)
        print("Will use (newly created) transposed song directory (output): " + transposedSongDirectory)

    # Query transposition
    globalHalfTones = int( query("Please specify half tones of transposition (7 or -5 for soprano ukelele and guitalele)","0") )
    print("Will use half tones of transposition: " + str(globalHalfTones))

    # Query transposition
    while True:
        yesNo = query('Apply capo/drop correction (confirm with "y" or "yes" without quotes)?','yes')
        if yesNo == "yes" or yesNo == "y":
            print("Will apply capo/drop correction")
            applyCapoDropCorrection = True
            break
        elif yesNo == "no" or yesNo == "n":
            print("Will not apply capo/drop correction")
            applyCapoDropCorrection = False
            break

    print("----------------------")

    for dirname, dirnames, filenames in os.walk( songDirectory ):
        for filename in sorted(filenames):
            songHalfTones = globalHalfTones
            #debug
            print filename
            print '*** songHalfTones:',songHalfTones
            name, extension = os.path.splitext(filename)
            songIn = open( os.path.join(dirname, filename) )
            songOut = open( os.path.join(transposedSongDirectory, filename), "w" )
            contents = songIn.read()
            contents = re.sub("\([^)]*\)", transpose, contents)
            songOut.write(contents)
            songOut.close()
            songIn.close()

<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2022 The Fiddle-Config Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Fiddle extensions to handle JAX code more elegantly.

Currently this just affects codegen, graphviz, and other debugging functions.
"""

from fiddle.codegen import import_manager
from fiddle.codegen import py_val_to_cst_converter
from fiddle.codegen import special_value_codegen
import jax
from jax import numpy as jnp

import libcst as cst

def _make_jnp_importable(name: str) -> special_value_codegen.Importable:
  return special_value_codegen.SingleImportable(
      "jax.numpy", lambda jnp_name: f"{jnp_name}.{name}")

_jnp_type_importables = (
    (jnp.bool_, _make_jnp_importable("bool_")),
    (jnp.uint8, _make_jnp_importable("uint8")),
    (jnp.uint16, _make_jnp_importable("uint16")),
    (jnp.uint32, _make_jnp_importable("uint32")),
    (jnp.uint64, _make_jnp_importable("uint64")),
    (jnp.int8, _make_jnp_importable("int8")),
    (jnp.int16, _make_jnp_importable("int16")),
    (jnp.int32, _make_jnp_importable("int32")),
    (jnp.int64, _make_jnp_importable("int64")),
    (jnp.bfloat16, _make_jnp_importable("bfloat16")),
    (jnp.float16, _make_jnp_importable("float16")),
    (jnp.float32, _make_jnp_importable("float32")),
    (jnp.float64, _make_jnp_importable("float64")),
    (jnp.complex64, _make_jnp_importable("complex64")),
    (jnp.complex128, _make_jnp_importable("complex128")),
)

_import_aliases = (
    # Rewrite internal import for JAX initializers.
    ("jax._src.nn.initializers", "from jax.nn import initializers"),
    ("jax.numpy", "from jax import numpy as jnp"),
)

def _make_jax_nn_importable(name: str) -> special_value_codegen.Importable:
  return special_value_codegen.SingleImportable(
      "jax", lambda jax_mod_name: f"{jax_mod_name}.nn.{name}")

_nn_type_importables = (
    (jax.nn.relu, _make_jax_nn_importable("relu")),
    (jax.nn.gelu, _make_jax_nn_importable("gelu")),
    (jax.nn.relu6, _make_jax_nn_importable("relu6")),
    (jax.nn.silu, _make_jax_nn_importable("silu")),
    (jax.nn.soft_sign, _make_jax_nn_importable("soft_sign")),
    (jax.nn.sigmoid, _make_jax_nn_importable("sigmoid")),
    (jax.nn.selu, _make_jax_nn_importable("selu")),
    (jax.nn.log_sigmoid, _make_jax_nn_importable("log_sigmoid")),
    (jax.nn.hard_tanh, _make_jax_nn_importable("hard_tanh")),
    (jax.nn.hard_swish, _make_jax_nn_importable("hard_swish")),
    (jax.nn.hard_silu, _make_jax_nn_importable("hard_silu")),
    (jax.nn.tanh, _make_jax_nn_importable("tanh")),
    (jax.nn.swish, _make_jax_nn_importable("swish")),
)

def is_jnp_device_array(value):
  """Returns true if `value` is a JAX numpy `DeviceArray`."""
  return isinstance(value, jnp.DeviceArray)

def convert_jnp_device_array_to_cst(value, convert_child):
  return cst.Call(
      func=cst.Attribute(value=convert_child(jnp), attr=cst.Name("array")),
      args=[
          cst.Arg(convert_child(value.tolist())),
          py_val_to_cst_converter.kwarg_to_cst("dtype",
                                               convert_child(value.dtype.name)),
          py_val_to_cst_converter.kwarg_to_cst("ndmin",
                                               convert_child(value.ndim))
      ])

def enable():
  """Registers JAX fiddle extensions.

  This allows for things like nicer handling of jax.numpy dtypes.
  """
  for value, importable in _jnp_type_importables:
    special_value_codegen.register_exact_value(value, importable)

  for value, importable in _nn_type_importables:
    special_value_codegen.register_exact_value(value, importable)

  for module_str, import_stmt in _import_aliases:
    import_manager.register_import_alias(module_str, import_stmt)

  # The odd calling syntax here ("register(type)(handler)") comes from the fact
  # that register_converter is usually a decorator, but we call it directly.
  py_val_to_cst_converter.register_py_val_to_cst_converter(is_jnp_device_array)(
      convert_jnp_device_array_to_cst)
<commit_msg>Use the new public jax.Array type.<commit_after># coding=utf-8
# Copyright 2022 The Fiddle-Config Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Fiddle extensions to handle JAX code more elegantly.

Currently this just affects codegen, graphviz, and other debugging functions.
"""

from fiddle.codegen import import_manager
from fiddle.codegen import py_val_to_cst_converter
from fiddle.codegen import special_value_codegen
import jax
from jax import numpy as jnp

import libcst as cst

def _make_jnp_importable(name: str) -> special_value_codegen.Importable:
  return special_value_codegen.SingleImportable(
      "jax.numpy", lambda jnp_name: f"{jnp_name}.{name}")

_jnp_type_importables = (
    (jnp.bool_, _make_jnp_importable("bool_")),
    (jnp.uint8, _make_jnp_importable("uint8")),
    (jnp.uint16, _make_jnp_importable("uint16")),
    (jnp.uint32, _make_jnp_importable("uint32")),
    (jnp.uint64, _make_jnp_importable("uint64")),
    (jnp.int8, _make_jnp_importable("int8")),
    (jnp.int16, _make_jnp_importable("int16")),
    (jnp.int32, _make_jnp_importable("int32")),
    (jnp.int64, _make_jnp_importable("int64")),
    (jnp.bfloat16, _make_jnp_importable("bfloat16")),
    (jnp.float16, _make_jnp_importable("float16")),
    (jnp.float32, _make_jnp_importable("float32")),
    (jnp.float64, _make_jnp_importable("float64")),
    (jnp.complex64, _make_jnp_importable("complex64")),
    (jnp.complex128, _make_jnp_importable("complex128")),
)

_import_aliases = (
    # Rewrite internal import for JAX initializers.
    ("jax._src.nn.initializers", "from jax.nn import initializers"),
    ("jax.numpy", "from jax import numpy as jnp"),
)

def _make_jax_nn_importable(name: str) -> special_value_codegen.Importable:
  return special_value_codegen.SingleImportable(
      "jax", lambda jax_mod_name: f"{jax_mod_name}.nn.{name}")

_nn_type_importables = (
    (jax.nn.relu, _make_jax_nn_importable("relu")),
    (jax.nn.gelu, _make_jax_nn_importable("gelu")),
    (jax.nn.relu6, _make_jax_nn_importable("relu6")),
    (jax.nn.silu, _make_jax_nn_importable("silu")),
    (jax.nn.soft_sign, _make_jax_nn_importable("soft_sign")),
    (jax.nn.sigmoid, _make_jax_nn_importable("sigmoid")),
    (jax.nn.selu, _make_jax_nn_importable("selu")),
    (jax.nn.log_sigmoid, _make_jax_nn_importable("log_sigmoid")),
    (jax.nn.hard_tanh, _make_jax_nn_importable("hard_tanh")),
    (jax.nn.hard_swish, _make_jax_nn_importable("hard_swish")),
    (jax.nn.hard_silu, _make_jax_nn_importable("hard_silu")),
    (jax.nn.tanh, _make_jax_nn_importable("tanh")),
    (jax.nn.swish, _make_jax_nn_importable("swish")),
)

def is_jnp_device_array(value):
  """Returns true if `value` is a JAX numpy `DeviceArray`."""
  return isinstance(value, jax.Array)

def convert_jnp_device_array_to_cst(value, convert_child):
  return cst.Call(
      func=cst.Attribute(value=convert_child(jnp), attr=cst.Name("array")),
      args=[
          cst.Arg(convert_child(value.tolist())),
          py_val_to_cst_converter.kwarg_to_cst("dtype",
                                               convert_child(value.dtype.name)),
          py_val_to_cst_converter.kwarg_to_cst("ndmin",
                                               convert_child(value.ndim))
      ])

def enable():
  """Registers JAX fiddle extensions.

  This allows for things like nicer handling of jax.numpy dtypes.
  """
  for value, importable in _jnp_type_importables:
    special_value_codegen.register_exact_value(value, importable)

  for value, importable in _nn_type_importables:
    special_value_codegen.register_exact_value(value, importable)

  for module_str, import_stmt in _import_aliases:
    import_manager.register_import_alias(module_str, import_stmt)

  # The odd calling syntax here ("register(type)(handler)") comes from the fact
  # that register_converter is usually a decorator, but we call it directly.
  py_val_to_cst_converter.register_py_val_to_cst_converter(is_jnp_device_array)(
      convert_jnp_device_array_to_cst)
<|endoftext|>


<commit_before>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2016
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
#    chen.wu@icrar.org

from optparse import OptionParser
import os, datetime
import sys
import threading
import traceback

from bottle import route, run, request, get, static_file, template, redirect, response, HTTPResponse

from dfms import droputils, restutils
from dfms.dropmake.pg_generator import LG, PGT, GraphException, MetisPGTP, PyrrosPGTP, MySarkarPGTP, MinNumPartsPGTP, PSOPGTP
from dfms.dropmake.pg_manager import PGManager
from dfms.dropmake.scheduler import SchedulerException
from dfms.manager.client import CompositeManagerClient
from dfms.restutils import RestClientException
import pkg_resources
import json

py   = sys.version_info
py3k = py >= (3, 0, 0)
if( py3k ):
    b2s = lambda b: b.decode('utf-8')
else:
    b2s = lambda s: s

#lg_dir = None
post_sem = threading.Semaphore(1)
gen_pgt_sem = threading.Semaphore(1)

err_prefix = "[Error]"
DEFAULT_LG_NAME = "cont_img.json"
DEFAULT_PGT_VIEW_NAME = "lofar_pgt-view.json"
MAX_PGT_FN_CNT= 300
pgt_fn_count = 0

def lg_path(lg_name):
    return "{0}/{1}".format(lg_dir, lg_name)

def lg_exists(lg_name):
    return os.path.exists(lg_path(lg_name))

def lg_repo_contents():
    # We currently allow only one depth level
    b = os.path.basename
    lgs = {b(dirpath):fnames for dirpath,_,fnames in os.walk(lg_dir) if dirpath != lg_dir}
    return lgs

@route('/static/<filepath:path>')
def server_static(filepath):
    staticRoot = pkg_resources.resource_filename(__name__, resource_name=".")  # @UndefinedVariable
    return static_file(filepath, root=staticRoot)

@route('/jsonbody', method='POST')
def jsonbody_post():
    """
    Post graph JSON representation to LG or PG manager
    """
    # see the table in http://bottlepy.org/docs/dev/tutorial.html#html-form-handling
    lg_name = request.POST['lg_name']
    if (lg_exists(lg_name)):
        lg_content = request.POST['lg_content']
        lg_path = "{0}/{1}".format(lg_dir, lg_name)
        post_sem.acquire()
        try:
            # overwrite file on disks
            # print "writing to {0}".format(lg_path)
            with open(lg_path, "w") as f:
                f.write(lg_content)
        except Exception as excmd2:
            response.status = 500
            return "Fail to save logical graph {0}:{1}".format(lg_name, str(excmd2))
        finally:
            post_sem.release()
    else:
        response.status = 404
        return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)

@get('/jsonbody')
def jsonbody_get():
    """
    Return JSON representation of the logical graph
    """
    #print "get jsonbody is called"
    lg_name = request.query.get('lg_name')
    if (lg_name is None or len(lg_name) == 0):
        redirect('/jsonbody?lg_name={0}'.format(DEFAULT_LG_NAME))
    if (lg_exists(lg_name)):
        #print "Loading {0}".format(lg_name)
        lgp = lg_path(lg_name)
        with open(lgp, "r") as f:
            data = f.read()
        return data
    else:
        response.status = 404
        return "{0}: JSON graph {1} not found\n".format(err_prefix, lg_name)

@get('/lg_editor')
def load_lg_editor():
    """
    Let the LG editor load the specified logical graph JSON representation
    """
    lg_name = request.query.get('lg_name')
    if (lg_name is None or len(lg_name) == 0):
        #lg_name = DEFAULT_LG_NAME
        all_lgs = lg_repo_contents()
        first_dir = next(iter(all_lgs))
        first_lg = first_dir + '/' + all_lgs[first_dir][0]
        redirect('/lg_editor?lg_name={0}'.format(first_lg))

    if (lg_exists(lg_name)):
        tpl = b2s(pkg_resources.resource_string(__name__, 'lg_editor.html')) # @UndefinedVariable
        all_lgs = lg_repo_contents();
        return template(tpl, lg_json_name=lg_name, all_lgs=json.dumps(all_lgs))
    else:
        response.status = 404
        return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)

@get('/pg_viewer')
def load_pg_viewer():
    """
    RESTful interface for loading the Physical Graph Viewer
    """
    pgt_name = request.query.get('pgt_view_name')
    if (pgt_name is None or len(pgt_name) == 0):
        redirect('/pg_viewer?pgt_view_name={0}'.format(DEFAULT_PGT_VIEW_NAME))

    if (lg_exists(pgt_name)):
        tpl = b2s(pkg_resources.resource_string(__name__, 'pg_viewer.html')) # @UndefinedVariable
        return template(tpl, pgt_view_json_name=pgt_name)
    else:
        response.status = 404
        return "{0}: physical graph template (view) {1} not found\n".format(err_prefix, pgt_name)

@get('/show_gantt_chart')
def show_gantt_chart():
    """
    Restful interface to show the gantt chart
    """
    pgt_id = request.query.get('pgt_id')
    tpl = b2s(pkg_resources.resource_string(__name__, 'matrix_vis.html')) # @UndefinedVariable
    return template(tpl, pgt_view_json_name=pgt_id, vis_action="pgt_gantt_chart")

@get('/pgt_gantt_chart')
def get_gantt_chart():
    """
    RESTful interface to retrieve a Gantt Chart matrix associated with a PGT
    """
    pgt_id = request.query.get('pgt_id')
    try:
        ret = pg_mgr.get_gantt_chart(pgt_id)
        return ret
    except GraphException as ge:
        response.status = 500
        return "Failt to get Gantt chart for {0}: {1}".format(pgt_id, ge)

@get('/show_schedule_mat')
def show_schedule_mat():
    """
    Restful interface to show the gantt chart
    """
    pgt_id = request.query.get('pgt_id')
    tpl = b2s(pkg_resources.resource_string(__name__, 'matrix_vis.html')) # @UndefinedVariable
    return template(tpl, pgt_view_json_name=pgt_id, vis_action="pgt_schedule_mat")

@get('/pgt_schedule_mat')
def get_schedule_mat():
    """
    RESTful interface to retrieve a list of schedule matrices
    associated with a PGT
    """
    pgt_id = request.query.get('pgt_id')
    try:
        ret = pg_mgr.get_schedule_matrices(pgt_id)
        return ret
    except GraphException as ge:
        response.status = "500 {0}".format(ge)
        return "Failt to get Gantt chart for {0}: {1}".format(pgt_id, ge)

@get('/gen_pg')
def gen_pg():
    """
    RESTful interface to convert a PGT(P) into PG by mapping
    PGT(P) onto a given set of available resources
    """
    pgt_id = request.query.get('pgt_id')
    pgtp = pg_mgr.get_pgt(pgt_id)
    if (pgtp is None):
        response.status = 404
        return "PGT(P) with id {0} not found in the Physical Graph Manager"

    mhost = request.query.get('dfms_mgr_host')
    if (mhost is None):
        response.status = 500
        return "Must specify DFMS manager host"
    try:
        mport = int(request.query.get('dfms_mgr_port'))
        mgr_client = CompositeManagerClient(host=mhost, port=mport, timeout=30)
        # 1. get a list of nodes
        node_list = mgr_client.nodes()
        # 2. mapping PGTP to resources (node list)
        pg_spec = pgtp.to_pg_spec(node_list, ret_str=False)
        dt = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%S.%f')
        ssid = "{0}_{1}".format(pgt_id.split('.json')[0].split('_pgt')[0], dt)
        mgr_client.create_session(ssid)
        #print "session created"
        mgr_client.append_graph(ssid, pg_spec)
        #print "graph appended"
        completed_uids = droputils.get_roots(pg_spec)
        mgr_client.deploy_session(ssid, completed_uids=completed_uids)
        #mgr_client.deploy_session(ssid, completed_uids=[])
        #print "session deployed"
        # 3. redirect to the master drop manager
        redirect("http://{0}:{1}/session?sessionId={2}".format(mhost, mport, ssid))
    except RestClientException as re:
        response.status = 500
        return "Fail to interact with DFMS Drop Manager: {0}".format(re)
    except HTTPResponse:
        raise
    except Exception as ex:
        trace_msg = traceback.format_exc()
        print(trace_msg)
        response.status = 500
        return "Fail to deploy physical graph: {0}".format(ex)

@get('/gen_pgt')
def gen_pgt():
    """
    RESTful interface for translating Logical Graphs to Physical Graphs
    """
    lg_name = request.query.get('lg_name')
    if (lg_exists(lg_name)):
        try:
            lg = LG(lg_path(lg_name))
            drop_list = lg.unroll_to_tpl()
            part = request.query.get('num_par')
            if (part is None):
                is_part = ''
                pgt = PGT(drop_list)
            else:
                is_part = 'Partition'
                par_label = request.query.get('par_label')
                algo = request.query.get('algo')
                if ('metis' == algo):
                    min_goal = int(request.query.get('min_goal'))
                    ptype = int(request.query.get('ptype'))
                    ufactor = 100 - int(request.query.get('max_load_imb')) + 1
                    if (ufactor <= 0):
                        ufactor = 1
                    pgt = MetisPGTP(drop_list, int(part), min_goal, par_label, ptype, ufactor)
                elif ('mysarkar' == algo):
                    mp = request.query.get('merge_par')
                    mpp = True if '1' == mp else False
                    pgt = MySarkarPGTP(drop_list, int(part), par_label, int(request.query.get('max_dop')), merge_parts=mpp)
                elif ('min_num_parts' == algo):
                    time_greedy = 1 - float(request.query.get('time_greedy')) / 100.0 # assuming between 1 to 100
                    pgt = MinNumPartsPGTP(drop_list, int(request.query.get('deadline')),
                    int(part), par_label, int(request.query.get('max_dop')), merge_parts=False, optimistic_factor=time_greedy)
                elif ('pso' == algo):
                    params = ['deadline', 'topk', 'swarm_size']
                    pars = [None, 30, 40]
                    for i, para in enumerate(params):
                        try:
                            pars[i] = int(request.query.get(para))
                        except:
                            continue
                    pgt = PSOPGTP(drop_list, par_label, int(request.query.get('max_dop')),
                    deadline=pars[0], topk=pars[1], swarm_size=pars[2])
                elif ('pyrros' == algo):
                    pgt = PyrrosPGTP(drop_list, int(part))
                else:
                    raise GraphException("Unknown partition algorithm: {0}".format(algo))

            pgt_id = pg_mgr.add_pgt(pgt, lg_name)
            part_info = pgt.get_partition_info()
            tpl = b2s(pkg_resources.resource_string(__name__, 'pg_viewer.html')) # @UndefinedVariable
            return template(tpl, pgt_view_json_name=pgt_id, partition_info=part_info, is_partition_page=is_part)
        except GraphException as ge:
            response.status = 500
            return "Invalid Logical Graph {1}: {0}".format(str(ge), lg_name)
        except SchedulerException as se:
            response.status = 500
            return "Graph scheduling exception {1}: {0}".format(str(se), lg_name)
        except Exception as exp:
            response.status = 500
            trace_msg = traceback.format_exc()
            print(trace_msg)
            return "Graph partition exception {1}: {0}".format(str(exp), lg_name)
    else:
        response.status = 404
        return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)

if __name__ == "__main__":
    """
    e.g. python lg_web -d /tmp/
    """
    parser = OptionParser()
    parser.add_option("-d", "--lgdir", action="store", type="string", dest="lg_path",
                          help="logical graph path (input)")
    parser.add_option("-d", "--pgtdir", action="store", type="string", dest="pgt_path",
                          help="physical graph template path (output)")
    parser.add_option("-p", "--port", action="store", type="int", dest="lg_port", default=8084,
                      help="logical graph editor port (8084 by default)")

    (options, args) = parser.parse_args()
    if options.lg_path is None or options.pgt_path is None:
        parser.print_help()
        sys.exit(1)
    elif not os.path.exists(options.lg_path):
        print("{0} does not exist.".format(options.lg_path))
        sys.exit(1)

    try:
        os.makedirs(options.pgt_path)
    except:
        pass

    global lg_dir, pg_mgr
    lg_dir = options.lg_path
    pg_mgr = PGManager(options.pgt_path)
    # Let's use tornado, since luigi already depends on it
    run(host="0.0.0.0", server='wsgiref', port=options.lg_port, debug=False,
        server_class=restutils.ThreadingWSGIServer, handler_class=restutils.LoggingWSGIRequestHandler)
<commit_msg>Proper option name<commit_after>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2016
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
#    chen.wu@icrar.org

from optparse import OptionParser
import os, datetime
import sys
import threading
import traceback

from bottle import route, run, request, get, static_file, template, redirect, response, HTTPResponse

from dfms import droputils, restutils
from dfms.dropmake.pg_generator import LG, PGT, GraphException, MetisPGTP, PyrrosPGTP, MySarkarPGTP, MinNumPartsPGTP, PSOPGTP
from dfms.dropmake.pg_manager import PGManager
from dfms.dropmake.scheduler import SchedulerException
from dfms.manager.client import CompositeManagerClient
from dfms.restutils import RestClientException
import pkg_resources
import json

py   = sys.version_info
py3k = py >= (3, 0, 0)
if( py3k ):
    b2s = lambda b: b.decode('utf-8')
else:
    b2s = lambda s: s

#lg_dir = None
post_sem = threading.Semaphore(1)
gen_pgt_sem = threading.Semaphore(1)

err_prefix = "[Error]"
DEFAULT_LG_NAME = "cont_img.json"
DEFAULT_PGT_VIEW_NAME = "lofar_pgt-view.json"
MAX_PGT_FN_CNT= 300
pgt_fn_count = 0

def lg_path(lg_name):
    return "{0}/{1}".format(lg_dir, lg_name)

def lg_exists(lg_name):
    return os.path.exists(lg_path(lg_name))

def lg_repo_contents():
    # We currently allow only one depth level
    b = os.path.basename
    lgs = {b(dirpath):fnames for dirpath,_,fnames in os.walk(lg_dir) if dirpath != lg_dir}
    return lgs

@route('/static/<filepath:path>')
def server_static(filepath):
    staticRoot = pkg_resources.resource_filename(__name__, resource_name=".")  # @UndefinedVariable
    return static_file(filepath, root=staticRoot)

@route('/jsonbody', method='POST')
def jsonbody_post():
    """
    Post graph JSON representation to LG or PG manager
    """
    # see the table in http://bottlepy.org/docs/dev/tutorial.html#html-form-handling
    lg_name = request.POST['lg_name']
    if (lg_exists(lg_name)):
        lg_content = request.POST['lg_content']
        lg_path = "{0}/{1}".format(lg_dir, lg_name)
        post_sem.acquire()
        try:
            # overwrite file on disks
            # print "writing to {0}".format(lg_path)
            with open(lg_path, "w") as f:
                f.write(lg_content)
        except Exception as excmd2:
            response.status = 500
            return "Fail to save logical graph {0}:{1}".format(lg_name, str(excmd2))
        finally:
            post_sem.release()
    else:
        response.status = 404
        return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)

@get('/jsonbody')
def jsonbody_get():
    """
    Return JSON representation of the logical graph
    """
    #print "get jsonbody is called"
    lg_name = request.query.get('lg_name')
    if (lg_name is None or len(lg_name) == 0):
        redirect('/jsonbody?lg_name={0}'.format(DEFAULT_LG_NAME))
    if (lg_exists(lg_name)):
        #print "Loading {0}".format(lg_name)
        lgp = lg_path(lg_name)
        with open(lgp, "r") as f:
            data = f.read()
        return data
    else:
        response.status = 404
        return "{0}: JSON graph {1} not found\n".format(err_prefix, lg_name)

@get('/lg_editor')
def load_lg_editor():
    """
    Let the LG editor load the specified logical graph JSON representation
    """
    lg_name = request.query.get('lg_name')
    if (lg_name is None or len(lg_name) == 0):
        #lg_name = DEFAULT_LG_NAME
        all_lgs = lg_repo_contents()
        first_dir = next(iter(all_lgs))
        first_lg = first_dir + '/' + all_lgs[first_dir][0]
        redirect('/lg_editor?lg_name={0}'.format(first_lg))

    if (lg_exists(lg_name)):
        tpl = b2s(pkg_resources.resource_string(__name__, 'lg_editor.html')) # @UndefinedVariable
        all_lgs = lg_repo_contents();
        return template(tpl, lg_json_name=lg_name, all_lgs=json.dumps(all_lgs))
    else:
        response.status = 404
        return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)

@get('/pg_viewer')
def load_pg_viewer():
    """
    RESTful interface for loading the Physical Graph Viewer
    """
    pgt_name = request.query.get('pgt_view_name')
    if (pgt_name is None or len(pgt_name) == 0):
        redirect('/pg_viewer?pgt_view_name={0}'.format(DEFAULT_PGT_VIEW_NAME))

    if (lg_exists(pgt_name)):
        tpl = b2s(pkg_resources.resource_string(__name__, 'pg_viewer.html')) # @UndefinedVariable
        return template(tpl, pgt_view_json_name=pgt_name)
    else:
        response.status = 404
        return "{0}: physical graph template (view) {1} not found\n".format(err_prefix, pgt_name)

@get('/show_gantt_chart')
def show_gantt_chart():
    """
    Restful interface to show the gantt chart
    """
    pgt_id = request.query.get('pgt_id')
    tpl = b2s(pkg_resources.resource_string(__name__, 'matrix_vis.html')) # @UndefinedVariable
    return template(tpl, pgt_view_json_name=pgt_id, vis_action="pgt_gantt_chart")

@get('/pgt_gantt_chart')
def get_gantt_chart():
    """
    RESTful interface to retrieve a Gantt Chart matrix associated with a PGT
    """
    pgt_id = request.query.get('pgt_id')
    try:
        ret = pg_mgr.get_gantt_chart(pgt_id)
        return ret
    except GraphException as ge:
        response.status = 500
        return "Failt to get Gantt chart for {0}: {1}".format(pgt_id, ge)

@get('/show_schedule_mat')
def show_schedule_mat():
    """
    Restful interface to show the gantt chart
    """
    pgt_id = request.query.get('pgt_id')
    tpl = b2s(pkg_resources.resource_string(__name__, 'matrix_vis.html')) # @UndefinedVariable
    return template(tpl, pgt_view_json_name=pgt_id, vis_action="pgt_schedule_mat")

@get('/pgt_schedule_mat')
def get_schedule_mat():
    """
    RESTful interface to retrieve a list of schedule matrices
    associated with a PGT
    """
    pgt_id = request.query.get('pgt_id')
    try:
        ret = pg_mgr.get_schedule_matrices(pgt_id)
        return ret
    except GraphException as ge:
        response.status = "500 {0}".format(ge)
        return "Failt to get Gantt chart for {0}: {1}".format(pgt_id, ge)

@get('/gen_pg')
def gen_pg():
    """
    RESTful interface to convert a PGT(P) into PG by mapping
    PGT(P) onto a given set of available resources
    """
    pgt_id = request.query.get('pgt_id')
    pgtp = pg_mgr.get_pgt(pgt_id)
    if (pgtp is None):
        response.status = 404
        return "PGT(P) with id {0} not found in the Physical Graph Manager"

    mhost = request.query.get('dfms_mgr_host')
    if (mhost is None):
        response.status = 500
        return "Must specify DFMS manager host"
    try:
        mport = int(request.query.get('dfms_mgr_port'))
        mgr_client = CompositeManagerClient(host=mhost, port=mport, timeout=30)
        # 1. get a list of nodes
        node_list = mgr_client.nodes()
        # 2. mapping PGTP to resources (node list)
        pg_spec = pgtp.to_pg_spec(node_list, ret_str=False)
        dt = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%S.%f')
        ssid = "{0}_{1}".format(pgt_id.split('.json')[0].split('_pgt')[0], dt)
        mgr_client.create_session(ssid)
        #print "session created"
        mgr_client.append_graph(ssid, pg_spec)
        #print "graph appended"
        completed_uids = droputils.get_roots(pg_spec)
        mgr_client.deploy_session(ssid, completed_uids=completed_uids)
        #mgr_client.deploy_session(ssid, completed_uids=[])
        #print "session deployed"
        # 3. redirect to the master drop manager
        redirect("http://{0}:{1}/session?sessionId={2}".format(mhost, mport, ssid))
    except RestClientException as re:
        response.status = 500
        return "Fail to interact with DFMS Drop Manager: {0}".format(re)
    except HTTPResponse:
        raise
    except Exception as ex:
        trace_msg = traceback.format_exc()
        print(trace_msg)
        response.status = 500
        return "Fail to deploy physical graph: {0}".format(ex)

@get('/gen_pgt')
def gen_pgt():
    """
    RESTful interface for translating Logical Graphs to Physical Graphs
    """
    lg_name = request.query.get('lg_name')
    if (lg_exists(lg_name)):
        try:
            lg = LG(lg_path(lg_name))
            drop_list = lg.unroll_to_tpl()
            part = request.query.get('num_par')
            if (part is None):
                is_part = ''
                pgt = PGT(drop_list)
            else:
                is_part = 'Partition'
                par_label = request.query.get('par_label')
                algo = request.query.get('algo')
                if ('metis' == algo):
                    min_goal = int(request.query.get('min_goal'))
                    ptype = int(request.query.get('ptype'))
                    ufactor = 100 - int(request.query.get('max_load_imb')) + 1
                    if (ufactor <= 0):
                        ufactor = 1
                    pgt = MetisPGTP(drop_list, int(part), min_goal, par_label, ptype, ufactor)
                elif ('mysarkar' == algo):
                    mp = request.query.get('merge_par')
                    mpp = True if '1' == mp else False
                    pgt = MySarkarPGTP(drop_list, int(part), par_label, int(request.query.get('max_dop')), merge_parts=mpp)
                elif ('min_num_parts' == algo):
                    time_greedy = 1 - float(request.query.get('time_greedy')) / 100.0 # assuming between 1 to 100
                    pgt = MinNumPartsPGTP(drop_list, int(request.query.get('deadline')),
                    int(part), par_label, int(request.query.get('max_dop')), merge_parts=False, optimistic_factor=time_greedy)
                elif ('pso' == algo):
                    params = ['deadline', 'topk', 'swarm_size']
                    pars = [None, 30, 40]
                    for i, para in enumerate(params):
                        try:
                            pars[i] = int(request.query.get(para))
                        except:
                            continue
                    pgt = PSOPGTP(drop_list, par_label, int(request.query.get('max_dop')),
                    deadline=pars[0], topk=pars[1], swarm_size=pars[2])
                elif ('pyrros' == algo):
                    pgt = PyrrosPGTP(drop_list, int(part))
                else:
                    raise GraphException("Unknown partition algorithm: {0}".format(algo))

            pgt_id = pg_mgr.add_pgt(pgt, lg_name)
            part_info = pgt.get_partition_info()
            tpl = b2s(pkg_resources.resource_string(__name__, 'pg_viewer.html')) # @UndefinedVariable
            return template(tpl, pgt_view_json_name=pgt_id, partition_info=part_info, is_partition_page=is_part)
        except GraphException as ge:
            response.status = 500
            return "Invalid Logical Graph {1}: {0}".format(str(ge), lg_name)
        except SchedulerException as se:
            response.status = 500
            return "Graph scheduling exception {1}: {0}".format(str(se), lg_name)
        except Exception as exp:
            response.status = 500
            trace_msg = traceback.format_exc()
            print(trace_msg)
            return "Graph partition exception {1}: {0}".format(str(exp), lg_name)
    else:
        response.status = 404
        return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)

if __name__ == "__main__":
    """
    e.g. python lg_web -d /tmp/
    """
    parser = OptionParser()
    parser.add_option("-d", "--lgdir", action="store", type="string", dest="lg_path",
                          help="logical graph path (input)")
    parser.add_option("-t", "--pgtdir", action="store", type="string", dest="pgt_path",
                          help="physical graph template path (output)")
    parser.add_option("-p", "--port", action="store", type="int", dest="lg_port", default=8084,
                      help="logical graph editor port (8084 by default)")

    (options, args) = parser.parse_args()
    if options.lg_path is None or options.pgt_path is None:
        parser.print_help()
        sys.exit(1)
    elif not os.path.exists(options.lg_path):
        print("{0} does not exist.".format(options.lg_path))
        sys.exit(1)

    try:
        os.makedirs(options.pgt_path)
    except:
        pass

    global lg_dir, pg_mgr
    lg_dir = options.lg_path
    pg_mgr = PGManager(options.pgt_path)
    # Let's use tornado, since luigi already depends on it
    run(host="0.0.0.0", server='wsgiref', port=options.lg_port, debug=False,
        server_class=restutils.ThreadingWSGIServer, handler_class=restutils.LoggingWSGIRequestHandler)
<|endoftext|>


<commit_before>##### Flask
from framework.flask import send_file, secure_filename, app, \
    redirect, request, url_for, send_from_directory, \
    Blueprint, render_template, render_template_string, jsonify, abort, \
    make_response

##### ODM
from modularodm import FlaskStoredObject as StoredObject, fields, storage
from modularodm.query.querydialect import DefaultQueryDialect as Q

###### Mongo
from framework.mongo import db, set_up_storage

##### Sessions
from framework.sessions import goback, set_previous_url, session, create_session

##### Exceptions
from exceptions import HTTPError

##### Routing
from framework.routing import (Rule, process_rules,
                               WebRenderer, json_renderer,
                               render_mako_string)

###### Auth
from framework.auth import get_current_username, get_current_user_id, get_user, \
    get_current_user, must_be_logged_in, User

##### Form
from framework.forms import Form, PasswordField, BooleanField, IntegerField, \
    DateField, DateTimeField, FileField, HiddenField, RadioField, SelectField,\
    SelectMultipleField, SubmitField, TextAreaField, TextField, validators, \
    push_errors_to_status, MyTextInput, FieldList

##### Search
from framework.search import generate_keywords, search

##### Email
from framework.email.tasks import send_email

##### Status
from framework.status import push_status_message

##### Analytics
from framework.analytics import update_counters, get_basic_counters

import pytz

def convert_datetime(date, to='US/Eastern'):
    to_zone = pytz.timezone(to)
    date = date.replace(tzinfo=pytz.utc)
    return to_zone.normalize(date.astimezone(to_zone))
<commit_msg>Remove send_email from framework/__init__.py<commit_after>##### Flask
from framework.flask import send_file, secure_filename, app, \
    redirect, request, url_for, send_from_directory, \
    Blueprint, render_template, render_template_string, jsonify, abort, \
    make_response

##### ODM
from modularodm import FlaskStoredObject as StoredObject, fields, storage
from modularodm.query.querydialect import DefaultQueryDialect as Q

###### Mongo
from framework.mongo import db, set_up_storage

##### Sessions
from framework.sessions import goback, set_previous_url, session, create_session

##### Exceptions
from exceptions import HTTPError

##### Routing
from framework.routing import (Rule, process_rules,
                               WebRenderer, json_renderer,
                               render_mako_string)

###### Auth
from framework.auth import get_current_username, get_current_user_id, get_user, \
    get_current_user, must_be_logged_in, User

##### Form
from framework.forms import Form, PasswordField, BooleanField, IntegerField, \
    DateField, DateTimeField, FileField, HiddenField, RadioField, SelectField,\
    SelectMultipleField, SubmitField, TextAreaField, TextField, validators, \
    push_errors_to_status, MyTextInput, FieldList

##### Search
from framework.search import generate_keywords, search

##### Status
from framework.status import push_status_message

##### Analytics
from framework.analytics import update_counters, get_basic_counters

import pytz

def convert_datetime(date, to='US/Eastern'):
    to_zone = pytz.timezone(to)
    date = date.replace(tzinfo=pytz.utc)
    return to_zone.normalize(date.astimezone(to_zone))
<|endoftext|>


<commit_before><commit_msg>BUG : Fixed a bug in msct_image : The bug will be detailed in issue 503 and officially fixed there<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python
"""
created on Wed Jan 30 2018
author: Christian Olaf Haeusler

To Do:
    argparser
"""
from __future__ import print_function
from collections import defaultdict
from datetime import datetime
import csv
import os
import re
import sys

# constants #
MOVIE = True
CROPPED = 0 # in sec; is a concatenated time series with cropped volumes used?
INPUT_FILES = ['structure.csv',
               'speech_vocalization.csv',
               'speech_google_narrator.csv'] # sys.argv[1]
OUT_DIR = './annos_segmented/output'

SEGMENTS_OFFSETS = (
    (0.00, 0.00),
    (886.00, 0.00),
    (1752.08, 0.08),  # third segment's start
    (2612.16, 0.16),
    (3572.20, 0.20),
    (4480.28, 0.28),
    (5342.36, 0.36),
    (6410.44, 0.44),  # last segment's start
    (7086.00, 0.00))  # movie's last time point

# functions #
def read_anno(anno):
    '''not pretty but works with different kind of annoation formats
    '''
    with open(anno, 'r') as txt_file:
        rows = txt_file.readlines()

    print('\nReading:', os.path.basename(anno))

    cleaned = []
    for row in rows:
        row = row.strip()
        row = row.split(',')

        # skip the header
        if row[0] in ['time', 'start']:
            continue

        # skip column 1 if it cointains '???' (s.speech_google_narrator.csv)
        if '???' in row[1]:
            continue

        # in column 0 and (maybe) column 1, convert time stamps (hh:mm:ss:ff)
        # to seconds (s. structure.csv and speech_vocalization.csv)
        regex = r'[\d#]+:[\d#]+:[\d#]+:[\d#]+'

        # check if time info is given as a time stamp in column 0
        # and if the time stamp does not contain a commentary ('#')
        if re.match(regex, row[0]):
            if '#' not in row[0]:
                row[0] = time_stamp_to_msec(row[0]) / 1000.0
            else:
                print('skipping', row)
                continue
        # else time must be given in seconds already
        else:
            row[0] = float(row[0])

        # check if column 1 gives time info, too ('end')
        # check if it is given as a time stamp (hh:mm:ss:ff)
        # and if the time stamp contains a commentary
        if re.match(regex, row[1]):
            if '#' not in row[1]:
                row[1] = time_stamp_to_msec(row[1]) / 1000.0
            else:
                print('skipping', row)
        # if it is not a time stamp it must be in seconds
        # or the column does not provide time info at all
        else:
            try:
                row[1] = float(row[1])
            except ValueError as e:
                pass

        cleaned.append(row)

    return cleaned

def time_stamp_to_msec(t_stamp='01:50:34:01'):
    '''
    Input:
        time stamp (str) in format HH:MM:SS:Frame

    Output:
        time point in milliseconds (int)
    '''
    splitted_stamp = t_stamp.split(':')
    milliseconds = (int(splitted_stamp[0]) * 60 * 60 * 1000) +\
                   (int(splitted_stamp[1]) * 60 * 1000) +\
                   (int(splitted_stamp[2]) * 1000) +\
                   (int(splitted_stamp[3]) * 40)

    return milliseconds

def msec_to_time_stamp(milliseconds=6634040):
    '''
    Input:
        a time point in  milliseconds (int)

    Output:
        a time stamp (str) in format HH:MM:SS:Frame
    '''
    # convert in case function was called from the command line with the
    # timing given as a string
    milliseconds = int(milliseconds)

    hours = (milliseconds / (60 * 60 * 1000))
    minutes = (milliseconds % (60 * 60 * 1000) / (60 * 1000))
    seconds = (milliseconds % (60 * 60 * 1000) % (60 * 1000) / 1000)
    frame = (milliseconds % (60 * 60 * 1000) % (60 * 1000) % (1000) // 40)
    time_stamp = '%02d:%02d:%02d:%02d' % (hours, minutes, seconds, frame)

    return time_stamp

def get_run_number(starts, onset):
    '''
    '''
    for start in sorted(starts, reverse=True):
        if onset >= start:
            run = starts.index(start)
            break

    return run

def fix_segment_shift(timing_in_anno, cropped_time):
    '''
    the function is not necessary anymore since the correction
    is implicitly done by additionally given offsets in SEGMENTS_OFFSETS

    fixes the timing of the 8 stimulus movie sigments
    https://github.com/psychoinformatics-de/studyforrest-data-phase2/blob/master/code/stimulus/movie/segment_timing.csv
    '''
    # regular case which will be kept in runs 1 and 2
    timing_in_segment = timing_in_anno

    # correct for the accumulating offsets in segments 3 to 8
    for segment_start, offset in sorted(SEGMENTS_OFFSETS, reverse = True):
        # if timing is in a critical segment, correct the timing
        if timing_in_anno >= segment_start + cropped_time:
            timing_in_segment = round(timing_in_anno - offset, 3)
            break

    return timing_in_segment

def fix_audio_timing(uncorrected_audio):
    '''the movie's audiotrack lacks behind the visual frames
    there is an slightly increasing offset (but problably no continuous drift)
    over the movie segments
    '''
    corrected_audio = uncorrected_audio
    return corrected_audio

def anno_time_to_seg_time(seg_starts, run_nr, anno_time, cropped_time):
    '''
    "The position of an event from a movie annotation with respect to the
    cropped fMRI time series can now be determined by substracting the
    start time of the respective segment as listed in Table 1"
    http://studyforrest.org/annotation_timing.html

    events occur earlier in the cropped stimulus segments.
    hence the cropped ammount is additionally substracted from the anno timing
    '''
    seg_time = round(anno_time - (seg_starts[run_nr] + cropped_time), 2)

    return seg_time

def write_segmented_annos(source_anno, movie, cropped, run_dict, out_dir, ):
    '''
    '''
    if MOVIE is True:
        stimulus = 'movie'
    else:
        stimulus = 'audio'

    old_anno_name = os.path.splitext(os.path.basename(source_anno))[0]
    new_anno_name = '%s_%s_%scr' % ((old_anno_name, stimulus, cropped))

    print('Writing results to %s' % new_anno_name)

    for run in sorted(run_dict.keys()):
        print(run)

        tnow = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
        out_fname = '%s_%s_run_%s.csv' % (new_anno_name, tnow, run + 1)
        out_path = os.path.join(out_dir, out_fname)
        print(out_path)

        # in case the OUT_DIR changes to a directory incl. subdirectories
        path = os.path.dirname(out_path)
        if not os.path.exists(path):
            os.makedirs(path)

        with open(out_path, 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerows(run_dict[run])

#### main program #####
if __name__ == "__main__":

    # read the annotation file
    for input_file in INPUT_FILES[:1]:
        anno = read_anno(input_file)
        segment_starts = [start for start, offset in SEGMENTS_OFFSETS]

        run_events = defaultdict(list)
        for row in anno:
        # get the run number
            run = get_run_number(segment_starts, row[0])

            # SEGMENT SHIFT correction
            # is now implicitly done by func 'anno_time_to_seg_time'
            # using the adjusted segment starts (s. SEGMENTS_OFFSETS)
#             row[0] = fix_segment_shift(row[0], CROPPED)
#             if type(row[1]) == float:
#                 row[1] = fix_segment_shift(row[1], CROPPED)

            # finally convert the timings of the continouos annotation
            # to timings in respect to the start of the corresponding segment
            row[0] = anno_time_to_seg_time(segment_starts, run, row[0], CROPPED)
            if type(row[1]) == float:
                row[1] = anno_time_to_seg_time(segment_starts, run, row[1], CROPPED)

            # AUDIO TIMING (MOVIE) correction
            # Dialoge im Film kommen 1/2 frame spater als das Hoerspiel,
            # das einem frame (40ms) nach vorn gezogen wurde
            if MOVIE is True:
                pass

            # AUDIO TIMING (AUDIOBOOK) correction
            if MOVIE is False:
                pass

            # append that shit
            run_events[run].append(row)

        write_segmented_annos(input_file, MOVIE, CROPPED, run_events, OUT_DIR)
<commit_msg>RF: Segment splitter to use Pandas and work based on cmdline arguments<commit_after>#!/usr/bin/python
"""
created on Wed Jan 30 2018
author: Christian Olaf Haeusler

To Do:
    argparser
"""
from __future__ import print_function
from collections import defaultdict
import os
from os.path import basename
from os.path import join as opj
from os.path import exists
import re
import sys
import pandas as pd

# constants #
MOVIE = True
CROPPED = 0 # in sec; is a concatenated time series with cropped volumes used?
INPUT_FILES = sys.argv[1:]
OUT_DIR = 'segments'

SEGMENTS_OFFSETS = (
    (0.00, 0.00),
    (886.00, 0.00),
    (1752.08, 0.08),  # third segment's start
    (2612.16, 0.16),
    (3572.20, 0.20),
    (4480.28, 0.28),
    (5342.36, 0.36),
    (6410.44, 0.44),  # last segment's start
    (7086.00, 0.00))  # movie's last time point

# functions #
def read_anno(anno):
    '''not pretty but works with different kind of annoation formats
    '''
    with open(anno, 'r') as txt_file:
        rows = txt_file.readlines()

    print('\nReading:', os.path.basename(anno))

    cleaned = []
    for row in rows:
        row = row.strip()
        row = row.split(',')

        # skip the header
        if row[0] in ['time', 'start']:
            continue

        # skip column 1 if it cointains '???' (s.speech_google_narrator.csv)
        if '???' in row[1]:
            continue

        # in column 0 and (maybe) column 1, convert time stamps (hh:mm:ss:ff)
        # to seconds (s. structure.csv and speech_vocalization.csv)
        regex = r'[\d#]+:[\d#]+:[\d#]+:[\d#]+'

        # check if time info is given as a time stamp in column 0
        # and if the time stamp does not contain a commentary ('#')
        if re.match(regex, row[0]):
            if '#' not in row[0]:
                row[0] = time_stamp_to_msec(row[0]) / 1000.0
            else:
                print('skipping', row)
                continue
        # else time must be given in seconds already
        else:
            row[0] = float(row[0])

        # check if column 1 gives time info, too ('end')
        # check if it is given as a time stamp (hh:mm:ss:ff)
        # and if the time stamp contains a commentary
        if re.match(regex, row[1]):
            if '#' not in row[1]:
                row[1] = time_stamp_to_msec(row[1]) / 1000.0
            else:
                print('skipping', row)
        # if it is not a time stamp it must be in seconds
        # or the column does not provide time info at all
        else:
            try:
                row[1] = float(row[1])
            except ValueError as e:
                pass

        cleaned.append(row)

    return cleaned

def time_stamp_to_msec(t_stamp='01:50:34:01'):
    '''
    Input:
        time stamp (str) in format HH:MM:SS:Frame

    Output:
        time point in milliseconds (int)
    '''
    splitted_stamp = t_stamp.split(':')
    milliseconds = (int(splitted_stamp[0]) * 60 * 60 * 1000) +\
                   (int(splitted_stamp[1]) * 60 * 1000) +\
                   (int(splitted_stamp[2]) * 1000) +\
                   (int(splitted_stamp[3]) * 40)

    return milliseconds

def msec_to_time_stamp(milliseconds=6634040):
    '''
    Input:
        a time point in  milliseconds (int)

    Output:
        a time stamp (str) in format HH:MM:SS:Frame
    '''
    # convert in case function was called from the command line with the
    # timing given as a string
    milliseconds = int(milliseconds)

    hours = (milliseconds / (60 * 60 * 1000))
    minutes = (milliseconds % (60 * 60 * 1000) / (60 * 1000))
    seconds = (milliseconds % (60 * 60 * 1000) % (60 * 1000) / 1000)
    frame = (milliseconds % (60 * 60 * 1000) % (60 * 1000) % (1000) // 40)
    time_stamp = '%02d:%02d:%02d:%02d' % (hours, minutes, seconds, frame)

    return time_stamp

def get_run_number(starts, onset):
    '''
    '''
    for start in sorted(starts, reverse=True):
        if onset >= start:
            run = starts.index(start)
            break

    return run

def fix_segment_shift(timing_in_anno, cropped_time):
    '''
    the function is not necessary anymore since the correction
    is implicitly done by additionally given offsets in SEGMENTS_OFFSETS

    fixes the timing of the 8 stimulus movie sigments
    https://github.com/psychoinformatics-de/studyforrest-data-phase2/blob/master/code/stimulus/movie/segment_timing.csv
    '''
    # regular case which will be kept in runs 1 and 2
    timing_in_segment = timing_in_anno

    # correct for the accumulating offsets in segments 3 to 8
    for segment_start, offset in sorted(SEGMENTS_OFFSETS, reverse = True):
        # if timing is in a critical segment, correct the timing
        if timing_in_anno >= segment_start + cropped_time:
            timing_in_segment = round(timing_in_anno - offset, 3)
            break

    return timing_in_segment

def fix_audio_timing(uncorrected_audio):
    '''the movie's audiotrack lacks behind the visual frames
    there is an slightly increasing offset (but problably no continuous drift)
    over the movie segments
    '''
    corrected_audio = uncorrected_audio
    return corrected_audio

def anno_time_to_seg_time(seg_starts, run_nr, anno_time, cropped_time):
    '''
    "The position of an event from a movie annotation with respect to the
    cropped fMRI time series can now be determined by substracting the
    start time of the respective segment as listed in Table 1"
    http://studyforrest.org/annotation_timing.html

    events occur earlier in the cropped stimulus segments.
    hence the cropped ammount is additionally substracted from the anno timing
    '''
    seg_time = round(anno_time - (seg_starts[run_nr] + cropped_time), 2)

    return seg_time

def write_segmented_annos(infilename, movie, cropped, run_dict, out_dir, ):
    '''
    '''
    if MOVIE is True:
        stimulus = 'avmovie'
    else:
        stimulus = 'aomovie'

    basefilename = basename(infilename)[:-4]
    outdir = opj(out_dir, stimulus)
    if not exists(outdir):
        os.makedirs(outdir)

    for run in sorted(run_dict.keys()):
        outname = opj(out_dir, stimulus, '{}_run-{}_events.tsv'.format(
            basefilename,
            run + 1))

        pd.DataFrame.from_records(
            run_dict[run],
            columns=run_dict[run][0].dtype.names).to_csv(
                outname,
                sep='\t',
                index=False)

#### main program #####
if __name__ == "__main__":

    # read the annotation file
    for input_file in INPUT_FILES[:1]:
        anno = pd.read_csv(input_file, sep='\t').to_records(index=False)
        segment_starts = [start for start, offset in SEGMENTS_OFFSETS]

        run_events = defaultdict(list)
        for row in anno:
            # get the run number
            run = get_run_number(segment_starts, row['onset'])

            # SEGMENT SHIFT correction
            # is now implicitly done by func 'anno_time_to_seg_time'
            # using the adjusted segment starts (s. SEGMENTS_OFFSETS)
#             row[0] = fix_segment_shift(row[0], CROPPED)
#             if type(row[1]) == float:
#                 row[1] = fix_segment_shift(row[1], CROPPED)

            # finally convert the timings of the continouos annotation
            # to timings in respect to the start of the corresponding segment
            onset = anno_time_to_seg_time(
                segment_starts,
                run,
                float(row['onset']),
                CROPPED)
            row['onset'] = onset

            # AUDIO TIMING (MOVIE) correction
            # Dialoge im Film kommen 1/2 frame spater als das Hoerspiel,
            # das einem frame (40ms) nach vorn gezogen wurde
            if MOVIE is True:
                pass

            # AUDIO TIMING (AUDIOBOOK) correction
            if MOVIE is False:
                pass

            # append that shit
            run_events[run].append(row)

        write_segmented_annos(input_file, MOVIE, CROPPED, run_events, OUT_DIR)
<|endoftext|>


<commit_before>import os
import sqlite3

import sys

from .simulation_plotter import SimulationPlotter
from .plotter import PlotData
from .constants import QUERIES

def print_error_message():
    print('Error using module plotter!')
    print('Try python -m plotter --help for more information')

def print_usage():
    print('Usage:')
    print('python -m plotter /path/to/database/results.db [simulation_A] [simulation_B]')
    print('\tPlots the results of the last simulation when both simulation_A and simulation_B are not provided.')
    print('\tPlots the results of simulation_A when just this parameter is provided.')
    print('\tPlots the results of simulation_A and simulation_B when both parameters are provided.')
    print('python -m plotter /path/to/database/results.db --sims')
    print('\tShows the list of simulations.')

if __name__ == '__main__':
    parameters = sys.argv[1:]

    if len(parameters) not in (1, 2, 3):
        print_error_message()
        sys.exit(1)

    if len(parameters) == 1 and parameters[0] == '--help':
        print_usage()
        sys.exit(0)

    path_to_database = parameters[0]

    if not os.path.isfile(path_to_database):
        print('Error: database does not exist!')
        sys.exit(2)

    connection = sqlite3.connect(path_to_database)
    cursor = connection.cursor()

    cursor.execute(QUERIES['select_sim'])

    if len(parameters) == 1:
        print("Plotting results of the last simulation...")
        simulation_name = cursor.fetchone()[0]
        controller = str(simulation_name).split('_')[0]

        cursor.execute(QUERIES['select_data'].format(simulation_name))
        plot_data = PlotData()

        for row in cursor.fetchall():
            plot_data.t.append(row[0])
            plot_data.x.append(row[1])
            plot_data.x_ref.append(row[2])
            plot_data.y.append(row[3])
            plot_data.y_ref.append(row[4])
            plot_data.theta.append(row[5])
            plot_data.theta_ref.append(row[6])
            plot_data.v_c.append(row[7])
            plot_data.w_c.append(row[8])
            plot_data.zeros.append(0)

        plotter = SimulationPlotter(plot_data, controller)
        plotter.plot_results()

    if len(parameters) == 2:
        rows = cursor.fetchall()
        if parameters[1] == '--sims':
            for row in rows:
                print(row[0])
            sys.exit(0)

    cursor.close()
    connection.close()
<commit_msg>feat: Report when simulation does not exists<commit_after>import os
import sqlite3

import sys

from .simulation_plotter import SimulationPlotter
from .plotter import PlotData
from .constants import QUERIES

def print_error_message():
    print('Error using module plotter!')
    print('Try python -m plotter --help for more information')

def print_usage():
    print('Usage:')
    print('python -m plotter /path/to/database/results.db [simulation_A] [simulation_B]')
    print('\tPlots the results of the last simulation when both simulation_A and simulation_B are not provided.')
    print('\tPlots the results of simulation_A when just this parameter is provided.')
    print('\tPlots the results of simulation_A and simulation_B when both parameters are provided.')
    print('python -m plotter /path/to/database/results.db --sims')
    print('\tShows the list of simulations.')

if __name__ == '__main__':
    parameters = sys.argv[1:]

    if len(parameters) not in (1, 2, 3):
        print_error_message()
        sys.exit(1)

    if len(parameters) == 1 and parameters[0] == '--help':
        print_usage()
        sys.exit(0)

    path_to_database = parameters[0]

    if not os.path.isfile(path_to_database):
        print('Error: database does not exist!')
        sys.exit(2)

    connection = sqlite3.connect(path_to_database)
    cursor = connection.cursor()

    cursor.execute(QUERIES['select_sim'])

    if len(parameters) == 1:
        print("Plotting results of the last simulation...")
        simulation_name = cursor.fetchone()[0]
        controller = str(simulation_name).split('_')[0]

        cursor.execute(QUERIES['select_data'].format(simulation_name))
        plot_data = PlotData()

        for row in cursor.fetchall():
            plot_data.t.append(row[0])
            plot_data.x.append(row[1])
            plot_data.x_ref.append(row[2])
            plot_data.y.append(row[3])
            plot_data.y_ref.append(row[4])
            plot_data.theta.append(row[5])
            plot_data.theta_ref.append(row[6])
            plot_data.v_c.append(row[7])
            plot_data.w_c.append(row[8])
            plot_data.zeros.append(0)

        plotter = SimulationPlotter(plot_data, controller)
        plotter.plot_results()

    if len(parameters) == 2:
        rows = cursor.fetchall()
        if parameters[1] == '--sims':
            for row in rows:
                print(row[0])
            sys.exit(0)

        if parameters[0] not in rows:
            print('Error: simulation does not exists')
            sys.exit(3)

    cursor.close()
    connection.close()
<|endoftext|>