<commit_before>"""
========================================================
Statistic the SV Stat after AGE Process
========================================================
Author : Shujia Huang & Siyang Liu
Date   : 2014-03-07 0idx:54:15
"""
import sys
import re
import os
import string
import numpy as np
import matplotlib.pyplot as plt

def DrawFig( figureFile, distance, properDepth, imProperDepth, nr, aa, bb, mscore, misprob, aveIden ) :

    fig = plt.figure( num=None, figsize=(16, 30), facecolor='w', edgecolor='k' )

    title  = ['Distance distribution', 'NRatio', 'Perfect Depth', 'Imperfect depth', '', '', '']
    ylabel = ['The position of breakpoint', 'N Ratio of varints', 'Perfect Depth', \
              'Both ImPerfect Depth', 'Map score', 'Mismapping Probability', 'Average Identity', 'ProperReadDepth', 'ImProperReadDepth']
    for i, data in enumerate ( [ distance, nr, aa, bb, mscore, misprob, aveIden, properDepth, imProperDepth ] ) :
        plt.subplot(9,2,2 * i + 1)
        #plt.title(title[i], fontsize=16)
        P = data[:,0] == 1; N = data[:,0] == 2; X = data[:,0] == 3
        plt.scatter(data[:,1][P], data[:,2][P], marker='o', c = 'g', alpha=0.3, linewidths = 0, label = 'Positive(%d)'%len(data[:,1][P]) ) # Positive
        plt.scatter(data[:,1][N], data[:,2][N], marker='o', c = 'r', alpha=0.3, linewidths = 0, label = 'Negative(%d)'%len(data[:,1][N]) ) # Negative
        plt.scatter(data[:,1][X], data[:,2][X], marker='o', c = 'Y', alpha=0.3, linewidths = 0, label = 'Positive->Negative(%d)' % len(data[:,1][X]) ) # Positive->Negative
        plt.legend(loc='upper left')
        plt.xlabel( 'Score'  , fontsize=16 )
        plt.ylabel( ylabel[i], fontsize=16 )

        plt.subplot(9, 2, 2*i + 2)
        NEW  = data[:,0] == 0
        good = data[:,1][NEW] >= 2.0
        bad  = data[:,1][NEW] < 2.0

        plt.scatter( data[:,1][NEW][good], data[:,2][NEW][good], marker='o', c = 'b', alpha=0.4, linewidths = 0, label = 'good(%d)' % len(data[:,1][NEW][good]) ) # good
        plt.scatter( data[:,1][NEW][bad] , data[:,2][NEW][bad] , marker='o', c = 'm', alpha=0.4, linewidths = 0, label = 'bad(%d)' % len( data[:,1][NEW][bad] ) ) # bad
        plt.legend(loc='upper left')
        plt.xlabel('Score'  , fontsize=16)

    fig.savefig(figureFile + '.png')
    fig.savefig(figureFile + '.pdf')

def DrawPhredScale ( figureFile, phredScal ) :

    fig = plt.figure()

    ylabel = ['Phred Scale']
    for i, data in enumerate ( [ phredScal ] ) :
        plt.subplot(2, 1, 2 * i + 1)
        P = data[:,0] == 1; N = data[:,0] == 2; X = data[:,0] == 3
        plt.scatter(data[:,1][P], data[:,2][P], marker='o', c = 'g', alpha=0.3, linewidths = 0, label = 'Positive(%d)'%len(data[:,1][P]) ) # Positive
        plt.scatter(data[:,1][N], data[:,2][N], marker='o', c = 'r', alpha=0.3, linewidths = 0, label = 'Negative(%d)'%len(data[:,1][N]) ) # Negative
        plt.scatter(data[:,1][X], data[:,2][X], marker='o', c = 'Y', alpha=0.3, linewidths = 0, label = 'Positive->Negative(%d)' % len(data[:,1][X]) ) # Positive->Negative
        plt.legend(loc='upper left')
        plt.ylabel( ylabel[i], fontsize=16 )

        plt.subplot(2, 1, 2*i + 2)
        NEW  = data[:,0] == 0
        good = data[:,1][NEW] >= 2.0
        bad  = data[:,1][NEW] < 2.0

        plt.scatter( data[:,1][NEW][good], data[:,2][NEW][good], marker='o', c = 'b', alpha=0.4, linewidths = 0, label = 'good(%d)' % len(data[:,1][NEW][good]) ) # good
        plt.scatter( data[:,1][NEW][bad] , data[:,2][NEW][bad] , marker='o', c = 'm', alpha=0.4, linewidths = 0, label = 'bad(%d)' % len( data[:,1][NEW][bad] ) ) # bad
        plt.legend(loc='upper left')
        plt.xlabel('Score'  , fontsize=16)
        plt.ylabel( ylabel[i], fontsize=16 )

    fig.savefig(figureFile + '.png')
    fig.savefig(figureFile + '.pdf')

def Accum ( data, isBig = False) :

    tmpD= data
    k   = sorted( tmpD.keys(), key = lambda d: float(d) )
    dat = []
    for i in range( len(k) ) :
        if isBig :
            for j in range(i,len(k)) : tmpD[k[i]][1] += tmpD[k[j]][0]
        else :
            for j in range(i+1) : tmpD[k[i]][1] += tmpD[k[j]][0]
        dat.append( [ float(k[i]), float(tmpD[k[i]][0]), float(tmpD[k[i]][1]) ] )

    return dat

def SampleFaLen ( faLenFile ) :

    if faLenFile[-3:] == '.gz' : I = os.popen( 'gzip -dc %s' % faLenFile )
    else                       : I = open( faLenFile )
    data = {}
    while 1 :
        lines = I.readlines ( 100000 )
        if not lines : break
        for line in lines :
            col = line.strip('\n').split()
            data[col[0]] = string.atoi( col[1] )
    I.close()
    return data

def LoadFaLen ( faLenLstFile ) :

    data = {}
    I    = open (faLenLstFile)
    for line in I.readlines() :

        if len( line.strip('\n').split() ) != 2: raise ValueError('[ERROR] The format of Fa length list maybe not right. It could just be : "sample FalenghtFile", but found',line)
        sampleId, fileName = line.strip('\n').split()
        if sampleId not in data : data[sampleId] = {}
        data[sampleId] = SampleFaLen( fileName )
    I.close()
    return data

def main ( argv ) :

    qFaLen    = LoadFaLen( argv[1] )
    figPrefix = 'test'
    if len(argv) > 2 : figPrefix = argv[2]
    if argv[0][-3:] == '.gz' :
        I = os.popen( 'gzip -dc %s' % argv[0] )
    else :
        I = open ( argv[0] )

    s, annotations, mark = set(), [], []
    print '#Chr\tPosition\tDistance\tLeftIden\tRightIden\tAveIden\tN-Ratio\tAA'
    while 1 : # VCF format

        lines = I.readlines( 100000 )
        if not lines : break

        for line in lines :

            col = line.strip('\n').split()
            if re.search( r'^#CHROM', line ) : col2sam = { i+9:sam for i,sam in enumerate(col[9:]) }
            if re.search(r'^#', line) : continue
            key = col[0] + ':' + col[1]
            if key in s : continue
            s.add(key)

            #if re.search(r'^PASS', col[6] ) : continue
            #if not re.search(r'_TRAIN_SITE', col[7]) : continue
            #if not re.search(r'^PASS', col[6] ) : continue

            fmat = { k:i for i,k in enumerate( col[8].split(':') ) }
            if 'VS' not in fmat or 'QR' not in fmat: continue
            if len(annotations) == 0 : annotations = [ [] for _ in col[9:] ]

            vcfinfo = { d.split('=')[0] : d.split('=')[1] for d in col[7].split(';') if len(d.split('=')) == 2 }
            vq      = string.atof( vcfinfo['VQ'] )
            if ('POSITIVE_TRAIN_SITE' in col[7]) and ('NEGATIVE_TRAIN_SITE' in col[7]) :
                mark.append( [3, vq] )
            elif 'POSITIVE_TRAIN_SITE' in col[7] :
                mark.append( [1, vq] )
            elif 'NEGATIVE_TRAIN_SITE' in col[7] :
                mark.append( [2, vq] )
            else :
                mark.append( [0, vq] )

            for i, sample in enumerate ( col[9:] ) :

                sampleId = col2sam[9+i]

                qr = sample.split(':')[fmat['QR']].split(',')[-1]
                if qr == '.' or sample.split(':')[fmat['MS']] == '.' :
                    annotations[i].append( [0, 0, 0, 0, 0, 0, 0, 0, 0] )
                    continue

                qId, qSta, qEnd = qr.split('-')
                qSta = string.atoi(qSta)
                qEnd = string.atoi(qEnd)

                if sampleId not in qFaLen : raise ValueError ('[ERROR] The sample name $s(in vcf) is not in the name of Fa list.' % sampleId )
                if qId not in qFaLen[sampleId] : raise ValueError ('[ERROR]', qId, 'is not been found in file', opt.qFalen, '\n' )
                qSta= int( qSta * 100 / qFaLen[sampleId][qId] + 0.5 )
                qEnd= int( qEnd * 100 / qFaLen[sampleId][qId] + 0.5 )
                if qSta > 100 or qEnd > 100 : raise ValueError ('[ERROR] Query size Overflow! sample : %s; scaffold : %s' % (sampleId, qId) )
                leg = qSta
                if 100 - qEnd < qSta : leg = qEnd
                nn  = string.atof(sample.split(':')[fmat['FN']])
                n   = round( 1000 * nn ) / 10.0                                  # N ratio
                alt = string.atoi( sample.split(':')[fmat['AA']].split(',')[1] ) # Alternate perfect
                bot = string.atoi( sample.split(':')[fmat['AA']].split(',')[3] ) # Both imperfect
                pro = string.atoi( sample.split(':')[fmat['RP']].split(',')[0] ) # Proper Pair
                ipr = string.atoi( sample.split(':')[fmat['RP']].split(',')[1] ) # ImProper Pair
                ms  = string.atoi( sample.split(':')[fmat['MS']]  )              # Mapping score
                mip = string.atof( sample.split(':')[fmat['MIP']] )              # Mismapping probability
                aveI= string.atoi( sample.split(':')[fmat['AE']].split(',')[3] ) # ave_iden in AGE
                annotations[i].append( [leg, n, alt, bot, pro, ipr, ms, mip, aveI] )
    I.close()
    print >> sys.stderr, '# Number of Positions: %d' % len( mark )
    if len( mark ) != len( annotations[0] ) : raise ValueError ('[ERROR] The size is not match mark=%d, annotations=%d!' % (len(mark), len(annotations)) )

    annotations = np.array( annotations );
    sampleNum   = len( annotations )

    """ Extractly we don't have do such normalization
    for i in range( sampleNum ) :
        if np.sum(annotations[i]) == 0: continue
        goodIndx  = [j for j, d in enumerate( annotations[i] ) if np.sum(d) > 0]
        mean      = np.array( [d for d in annotations[i] if np.sum(d) > 0] ).mean(axis=0)
        std       = np.array( [d for d in annotations[i] if np.sum(d) > 0] ).std (axis=0)
        annotations[i][goodIndx] = np.abs( (annotations[i][goodIndx] - mean)/std )  # Normalization Per sample
    """

    data, distance, properDepth, imProperDepth, nr, aa, bb, mscore, misprob, aveIden = [],[],[],[],[],[],[],[],[],[]
    phredScal = []
    for i in range( len(annotations[0]) ) :

        anno    = np.array( [ annotations[s][i] for s in range(sampleNum) if len(annotations[s][i][annotations[s][i]!=0]) > 0 ] ) # each person in the same position

        score  = np.array( [ annotations[s][i][-3] for s in range(sampleNum) if annotations[s][i][-3] > 0 ] )
        msprob = np.array( [ annotations[s][i][-2] for s in range(sampleNum) if annotations[s][i][-3] > 0 ] )
        phred  = -10 * np.log10( 1.0 - score.sum() / np.sum( score/(1.0 - msprob) ) ) # Phred scale

        if len( anno ) == 0 : continue
        leg, n, alt, bot, pro,ipr, ms, mip, aveI = np.median( anno, axis=0 )

        distance.append      ( [ mark[i][0], mark[i][1], leg ] )
        properDepth.append   ( [ mark[i][0], mark[i][1], pro ] )
        imProperDepth.append ( [ mark[i][0], mark[i][1], ipr ] )
        nr.append            ( [ mark[i][0], mark[i][1], n   ] )
        aa.append            ( [ mark[i][0], mark[i][1], alt ] )
        bb.append            ( [ mark[i][0], mark[i][1], bot ] )
        mscore.append        ( [ mark[i][0], mark[i][1], ms  ] )
        misprob.append       ( [ mark[i][0], mark[i][1], mip ] )
        aveIden.append       ( [ mark[i][0], mark[i][1], aveI] )
        phredScal.append     ( [ mark[i][0], mark[i][1], phred])

        data.append([leg, alt, pro,ipr, n, bot])
        print mark[i][0], mark[i][1], '\t', leg, '\t', pro, '\t', ipr,'\t', n, '\t', alt, '\t', bot

    data = np.array(data)
    print >> sys.stderr, '\nPosition\tALTernatePerfect\tLeftIdentity\tRightIdentity\tAveIden\tNRatio\tBothImperfect'
    print >> sys.stderr, 'Means: ', data.mean(axis=0), '\nstd  : ', data.std(axis=0), '\nMedian: ', np.median( data, axis=0 )
    print >> sys.stderr, '25 Percentile:', np.percentile(data, 25,axis=0), '\n50 Percentile:', np.percentile(data, 50,axis=0), '\n75 Percentile:', np.percentile(data, 75,axis=0)

    DrawFig( figPrefix, \
             np.array (distance      ), \
             np.array (properDepth   ), \
             np.array (imProperDepth ), \
             np.array (nr            ), \
             np.array (aa            ), \
             np.array (bb            ), \
             np.array (mscore        ), \
             np.array (misprob       ), \
             np.array (aveIden       )  )
    DrawPhredScale ( figPrefix + '.phred', np.array(phredScal) )

if __name__ == '__main__' :

    main(sys.argv[1:])

<commit_msg>set alpha = 0.05<commit_after>"""
========================================================
Statistic the SV Stat after AGE Process
========================================================
Author : Shujia Huang & Siyang Liu
Date   : 2014-03-07 0idx:54:15
"""
import sys
import re
import os
import string
import numpy as np
import matplotlib.pyplot as plt

def DrawFig( figureFile, distance, properDepth, imProperDepth, nr, aa, bb, mscore, misprob, aveIden ) :

    fig = plt.figure( num=None, figsize=(16, 30), facecolor='w', edgecolor='k' )

    title  = ['Distance distribution', 'NRatio', 'Perfect Depth', 'Imperfect depth', '', '', '']
    ylabel = ['The position of breakpoint', 'N Ratio of varints', 'Perfect Depth', \
              'Both ImPerfect Depth', 'Map score', 'Mismapping Probability', 'Average Identity', 'ProperReadDepth', 'ImProperReadDepth']
    for i, data in enumerate ( [ distance, nr, aa, bb, mscore, misprob, aveIden, properDepth, imProperDepth ] ) :
        plt.subplot(9,2,2 * i + 1)
        #plt.title(title[i], fontsize=16)
        P = data[:,0] == 1; N = data[:,0] == 2; X = data[:,0] == 3
        plt.scatter(data[:,1][P], data[:,2][P], marker='o', c = 'g', alpha=0.03, linewidths = 0, label = 'Positive(%d)'%len(data[:,1][P]) ) # Positive
        plt.scatter(data[:,1][N], data[:,2][N], marker='o', c = 'r', alpha=0.03, linewidths = 0, label = 'Negative(%d)'%len(data[:,1][N]) ) # Negative
        plt.scatter(data[:,1][X], data[:,2][X], marker='o', c = 'Y', alpha=0.03, linewidths = 0, label = 'Positive->Negative(%d)' % len(data[:,1][X]) ) # Positive->Negative
        plt.legend(loc='upper left')
        plt.xlabel( 'Score'  , fontsize=16 )
        plt.ylabel( ylabel[i], fontsize=16 )

        plt.subplot(9, 2, 2*i + 2)
        NEW  = data[:,0] == 0
        good = data[:,1][NEW] >= 2.0
        bad  = data[:,1][NEW] < 2.0

        plt.scatter( data[:,1][NEW][good], data[:,2][NEW][good], marker='o', c = 'b', alpha=0.03, linewidths = 0, label = 'good(%d)' % len(data[:,1][NEW][good]) ) # good
        plt.scatter( data[:,1][NEW][bad] , data[:,2][NEW][bad] , marker='o', c = 'm', alpha=0.03, linewidths = 0, label = 'bad(%d)' % len( data[:,1][NEW][bad] ) ) # bad
        plt.legend(loc='upper left')
        plt.xlabel('Score'  , fontsize=16)

    fig.savefig(figureFile + '.png')
    #fig.savefig(figureFile + '.pdf')

def DrawPhredScale ( figureFile, phredScal ) :

    fig = plt.figure()

    ylabel = ['Phred Scale']
    for i, data in enumerate ( [ phredScal ] ) :
        plt.subplot(2, 1, 2 * i + 1)
        P = data[:,0] == 1; N = data[:,0] == 2; X = data[:,0] == 3
        plt.scatter(data[:,1][N], data[:,2][N], marker='o', c = 'r', alpha=0.05, linewidths = 0, label = 'Negative(%d)'%len(data[:,1][N]) ) # Negative
        plt.scatter(data[:,1][P], data[:,2][P], marker='o', c = 'g', alpha=0.05, linewidths = 0, label = 'Positive(%d)'%len(data[:,1][P]) ) # Positive
        plt.scatter(data[:,1][X], data[:,2][X], marker='o', c = 'Y', alpha=0.05, linewidths = 0, label = 'Positive->Negative(%d)' % len(data[:,1][X]) ) # Positive->Negative
        plt.legend(loc='upper left')
        plt.ylabel( ylabel[i], fontsize=16 )

        plt.subplot(2, 1, 2*i + 2)
        NEW  = data[:,0] == 0
        good = data[:,1][NEW] >= 2.0
        bad  = data[:,1][NEW] < 2.0

        plt.scatter( data[:,1][NEW][bad] , data[:,2][NEW][bad] , marker='o', c = 'm', alpha=0.05, linewidths = 0, label = 'bad(%d)' % len( data[:,1][NEW][bad] ) ) # bad
        plt.scatter( data[:,1][NEW][good], data[:,2][NEW][good], marker='o', c = 'b', alpha=0.05, linewidths = 0, label = 'good(%d)' % len(data[:,1][NEW][good]) ) # good
        plt.legend(loc='upper left')
        plt.xlabel('Score'  , fontsize=16)
        plt.ylabel( ylabel[i], fontsize=16 )

    fig.savefig(figureFile + '.png')
    #fig.savefig(figureFile + '.pdf')

def Accum ( data, isBig = False) :

    tmpD= data
    k   = sorted( tmpD.keys(), key = lambda d: float(d) )
    dat = []
    for i in range( len(k) ) :
        if isBig :
            for j in range(i,len(k)) : tmpD[k[i]][1] += tmpD[k[j]][0]
        else :
            for j in range(i+1) : tmpD[k[i]][1] += tmpD[k[j]][0]
        dat.append( [ float(k[i]), float(tmpD[k[i]][0]), float(tmpD[k[i]][1]) ] )

    return dat

def SampleFaLen ( faLenFile ) :

    if faLenFile[-3:] == '.gz' : I = os.popen( 'gzip -dc %s' % faLenFile )
    else                       : I = open( faLenFile )
    data = {}
    while 1 :
        lines = I.readlines ( 100000 )
        if not lines : break
        for line in lines :
            col = line.strip('\n').split()
            data[col[0]] = string.atoi( col[1] )
    I.close()
    return data

def LoadFaLen ( faLenLstFile ) :

    data = {}
    I    = open (faLenLstFile)
    for line in I.readlines() :

        if len( line.strip('\n').split() ) != 2: raise ValueError('[ERROR] The format of Fa length list maybe not right. It could just be : "sample FalenghtFile", but found',line)
        sampleId, fileName = line.strip('\n').split()
        if sampleId not in data : data[sampleId] = {}
        data[sampleId] = SampleFaLen( fileName )
    I.close()
    return data

def main ( argv ) :

    qFaLen    = LoadFaLen( argv[1] )
    figPrefix = 'test'
    if len(argv) > 2 : figPrefix = argv[2]
    if argv[0][-3:] == '.gz' :
        I = os.popen( 'gzip -dc %s' % argv[0] )
    else :
        I = open ( argv[0] )

    s, annotations, mark = set(), [], []
    print '#Chr\tPosition\tDistance\tLeftIden\tRightIden\tAveIden\tN-Ratio\tAA'
    while 1 : # VCF format

        lines = I.readlines( 100000 )
        if not lines : break

        for line in lines :

            col = line.strip('\n').split()
            if re.search( r'^#CHROM', line ) : col2sam = { i+9:sam for i,sam in enumerate(col[9:]) }
            if re.search(r'^#', line) : continue
            key = col[0] + ':' + col[1]
            if key in s : continue
            s.add(key)

            #if re.search(r'^PASS', col[6] ) : continue
            #if not re.search(r'_TRAIN_SITE', col[7]) : continue
            #if not re.search(r'^PASS', col[6] ) : continue

            fmat = { k:i for i,k in enumerate( col[8].split(':') ) }
            if 'VS' not in fmat or 'QR' not in fmat: continue
            if len(annotations) == 0 : annotations = [ [] for _ in col[9:] ]

            vcfinfo = { d.split('=')[0] : d.split('=')[1] for d in col[7].split(';') if len(d.split('=')) == 2 }
            vq      = string.atof( vcfinfo['VQ'] )
            if ('POSITIVE_TRAIN_SITE' in col[7]) and ('NEGATIVE_TRAIN_SITE' in col[7]) :
                mark.append( [3, vq] )
            elif 'POSITIVE_TRAIN_SITE' in col[7] :
                mark.append( [1, vq] )
            elif 'NEGATIVE_TRAIN_SITE' in col[7] :
                mark.append( [2, vq] )
            else :
                mark.append( [0, vq] )

            for i, sample in enumerate ( col[9:] ) :

                sampleId = col2sam[9+i]

                qr = sample.split(':')[fmat['QR']].split(',')[-1]
                if qr == '.' or sample.split(':')[fmat['MS']] == '.' :
                    annotations[i].append( [0, 0, 0, 0, 0, 0, 0, 0, 0] )
                    continue

                qId, qSta, qEnd = qr.split('-')
                qSta = string.atoi(qSta)
                qEnd = string.atoi(qEnd)

                if sampleId not in qFaLen : raise ValueError ('[ERROR] The sample name $s(in vcf) is not in the name of Fa list.' % sampleId )
                if qId not in qFaLen[sampleId] : raise ValueError ('[ERROR]', qId, 'is not been found in file', opt.qFalen, '\n' )
                qSta= int( qSta * 100 / qFaLen[sampleId][qId] + 0.5 )
                qEnd= int( qEnd * 100 / qFaLen[sampleId][qId] + 0.5 )
                if qSta > 100 or qEnd > 100 : raise ValueError ('[ERROR] Query size Overflow! sample : %s; scaffold : %s' % (sampleId, qId) )
                leg = qSta
                if 100 - qEnd < qSta : leg = qEnd
                nn  = string.atof(sample.split(':')[fmat['FN']])
                n   = round( 1000 * nn ) / 10.0                                  # N ratio
                alt = string.atoi( sample.split(':')[fmat['AA']].split(',')[1] ) # Alternate perfect
                bot = string.atoi( sample.split(':')[fmat['AA']].split(',')[3] ) # Both imperfect
                pro = string.atoi( sample.split(':')[fmat['RP']].split(',')[0] ) # Proper Pair
                ipr = string.atoi( sample.split(':')[fmat['RP']].split(',')[1] ) # ImProper Pair
                ms  = string.atoi( sample.split(':')[fmat['MS']]  )              # Mapping score
                mip = string.atof( sample.split(':')[fmat['MIP']] )              # Mismapping probability
                aveI= string.atoi( sample.split(':')[fmat['AE']].split(',')[3] ) # ave_iden in AGE
                annotations[i].append( [leg, n, alt, bot, pro, ipr, ms, mip, aveI] )
    I.close()
    print >> sys.stderr, '# Number of Positions: %d' % len( mark )
    if len( mark ) != len( annotations[0] ) : raise ValueError ('[ERROR] The size is not match mark=%d, annotations=%d!' % (len(mark), len(annotations)) )

    annotations = np.array( annotations );
    sampleNum   = len( annotations )

    """ Extractly we don't have do such normalization
    for i in range( sampleNum ) :
        if np.sum(annotations[i]) == 0: continue
        goodIndx  = [j for j, d in enumerate( annotations[i] ) if np.sum(d) > 0]
        mean      = np.array( [d for d in annotations[i] if np.sum(d) > 0] ).mean(axis=0)
        std       = np.array( [d for d in annotations[i] if np.sum(d) > 0] ).std (axis=0)
        annotations[i][goodIndx] = np.abs( (annotations[i][goodIndx] - mean)/std )  # Normalization Per sample
    """

    data, distance, properDepth, imProperDepth, nr, aa, bb, mscore, misprob, aveIden = [],[],[],[],[],[],[],[],[],[]
    phredScal = []
    for i in range( len(annotations[0]) ) :

        anno    = np.array( [ annotations[s][i] for s in range(sampleNum) if len(annotations[s][i][annotations[s][i]!=0]) > 0 ] ) # each person in the same position

        score  = np.array( [ annotations[s][i][-3] for s in range(sampleNum) if annotations[s][i][-3] > 0 ] )
        msprob = np.array( [ annotations[s][i][-2] for s in range(sampleNum) if annotations[s][i][-3] > 0 ] )
        phred  = -10 * np.log10( 1.0 - score.sum() / np.sum( score/(1.0 - msprob) ) ) # Phred scale

        if len( anno ) == 0 : continue
        leg, n, alt, bot, pro,ipr, ms, mip, aveI = np.median( anno, axis=0 )

        distance.append      ( [ mark[i][0], mark[i][1], leg ] )
        properDepth.append   ( [ mark[i][0], mark[i][1], pro ] )
        imProperDepth.append ( [ mark[i][0], mark[i][1], ipr ] )
        nr.append            ( [ mark[i][0], mark[i][1], n   ] )
        aa.append            ( [ mark[i][0], mark[i][1], alt ] )
        bb.append            ( [ mark[i][0], mark[i][1], bot ] )
        mscore.append        ( [ mark[i][0], mark[i][1], ms  ] )
        misprob.append       ( [ mark[i][0], mark[i][1], mip ] )
        aveIden.append       ( [ mark[i][0], mark[i][1], aveI] )
        phredScal.append     ( [ mark[i][0], mark[i][1], phred])

        data.append([leg, alt, pro,ipr, n, bot])
        print mark[i][0], mark[i][1], '\t', leg, '\t', pro, '\t', ipr,'\t', n, '\t', alt, '\t', bot

    data = np.array(data)
    print >> sys.stderr, '\nPosition\tALTernatePerfect\tLeftIdentity\tRightIdentity\tAveIden\tNRatio\tBothImperfect'
    print >> sys.stderr, 'Means: ', data.mean(axis=0), '\nstd  : ', data.std(axis=0), '\nMedian: ', np.median( data, axis=0 )
    print >> sys.stderr, '25 Percentile:', np.percentile(data, 25,axis=0), '\n50 Percentile:', np.percentile(data, 50,axis=0), '\n75 Percentile:', np.percentile(data, 75,axis=0)

    DrawFig( figPrefix, \
             np.array (distance      ), \
             np.array (properDepth   ), \
             np.array (imProperDepth ), \
             np.array (nr            ), \
             np.array (aa            ), \
             np.array (bb            ), \
             np.array (mscore        ), \
             np.array (misprob       ), \
             np.array (aveIden       )  )
    DrawPhredScale ( figPrefix + '.phred', np.array(phredScal) )

if __name__ == '__main__' :

    main(sys.argv[1:])

<|endoftext|>


<commit_before>import heapq

class Trie(object):
  def __init__(self, data):
    self._smallest = float('inf')
    self._largest = 0
    # Trie's index (highest value characters first).
    self._index = {}
    self._len = 0
    values = []
    for key, value in data:
      if not self._largest:
        self._largest = value
      if value > self._smallest:
        raise AssertionError(
            'Items must be added to trie in descending order.')
      else:
        self._smallest = value
      self._add_to_index(key, value)
      values.append(value)
    num_values = len(values)
    if num_values % 2:
      self._median = (values[num_values // 2 + 1])
    else:
      self._median = (values[num_values // 2 - 1] + values[num_values // 2]) / 2

  def __contains__(self, key):
    result = self._find_prefix(key)
    return result is not None and result['match_weight'] > 0

  def __len__(self):
    return self._len

  def __getitem__(self, key):
    result = self._find_prefix(key)
    if result is None or result['match_weight'] == 0:
      raise KeyError('%s not in Trie' % key)
    return result['match_weight']

  def has_keys_with_prefix(self, prefix):
    return self._find_prefix(prefix) is not None

  def _find_prefix(self, prefix):
    l = len(prefix)
    pos = 0
    cursor = self._index
    while pos < l:
      target = prefix[pos]
      if target not in cursor:
        break
      cursor = cursor[target]
      pos += 1
    if pos == l:
      return cursor
    return None

  def magnitude(self):
    return self._largest

  def median(self):
    return self._median

  def walk(self, seek_sets):
    """Returns solutions matching `seek_sets`, ordered from high to low."""
    # TODO: Inline per findings from commit 47a736f.
    fringe = _MaxHeap()
    solutions = []
    acc = []
    fringe.push(float('inf'), (acc, self._index))
    stop_seek_pos = len(seek_sets) - 1
    while len(fringe):
      fringe_score = fringe.best_weight()
      acc, cursor = fringe.pop()
      pos = len(acc)
      seeking = seek_sets[pos]
      for c in seeking:
        if c not in cursor:
          continue
        acc.append(c)
        next_children = cursor[c]
        magnitude = next_children['max_weight']
        match_weight = next_children['match_weight']
        if match_weight:
          heapq.heappush(solutions, (-match_weight, ''.join(acc)))
        if len(next_children) > 2 and pos < stop_seek_pos:
          fringe.push(magnitude, (acc[:], next_children))
        acc.pop()
      while len(solutions) and -solutions[0][0] >= fringe_score:
        solution_weight, solution_word = heapq.heappop(solutions)
        yield solution_word, -solution_weight
    while len(solutions):
      solution_weight, solution_word = heapq.heappop(solutions)
      yield solution_word, -solution_weight

    if len(fringe._pool) > 10:
      print('WARNING')
      print('Max fringe size was: %s' % len(fringe._pool))

  def _add_to_index(self, word, weight):
    l = len(word)
    cursor = self._index
    pos = 0
    while pos < l:
      target = word[pos]
      if target not in cursor:
        break
      cursor = cursor[target]
      match_weight = cursor['match_weight']
      pos += 1
      if pos == l:
        if match_weight:
          # Duplicate addition.
          raise KeyError('%s already added' % word)
        # Exhausted input; this spells something now.
        cursor['match_weight'] = weight
        self._len += 1
        return
      elif len(cursor) <= 2:
        break  # End of Trie.
    # Add remaining characters.
    end = len(word) - 1
    for i in range(pos, len(word)):
      if i == end:
        cursor[word[i]] = {
          'max_weight': weight,
          'match_weight': weight,
        }
      else:
        new_cursor = {
          'max_weight': weight,
          'match_weight': 0,
        }
        cursor[word[i]] = new_cursor
        cursor = new_cursor
    self._len += 1

class _MaxHeap(object):
  def __init__(self):
    self._heap = []
    self._pool = []
    self._free_positions = []

  def __len__(self):
    return len(self._heap)

  def push(self, cost, o):
    if self._free_positions:
      idx = self._free_positions.pop()
      self._pool[idx] = o
    else:
      idx = len(self._pool)
      self._pool.append(o)
    heapq.heappush(self._heap, (-cost, idx))

  def best_weight(self):
    return -self._heap[0][0]

  def pop(self):
    _, idx = heapq.heappop(self._heap)
    result = self._pool[idx]
    self._pool[idx] = None
    self._free_positions.append(idx)
    return result
<commit_msg>Trivial change to formatting.<commit_after>import heapq

class Trie(object):
  def __init__(self, data):
    self._smallest = float('inf')
    self._largest = 0
    # Trie's index (highest value characters first).
    self._index = {}
    self._len = 0
    values = []
    for key, value in data:
      if not self._largest:
        self._largest = value
      if value > self._smallest:
        raise AssertionError(
            'Items must be added to trie in descending order.')
      else:
        self._smallest = value
      self._add_to_index(key, value)
      values.append(value)
    num_values = len(values)
    if num_values % 2:
      self._median = values[num_values // 2 + 1]
    else:
      self._median = (values[num_values // 2 - 1] + values[num_values // 2]) / 2

  def __contains__(self, key):
    result = self._find_prefix(key)
    return result is not None and result['match_weight'] > 0

  def __len__(self):
    return self._len

  def __getitem__(self, key):
    result = self._find_prefix(key)
    if result is None or result['match_weight'] == 0:
      raise KeyError('%s not in Trie' % key)
    return result['match_weight']

  def has_keys_with_prefix(self, prefix):
    return self._find_prefix(prefix) is not None

  def _find_prefix(self, prefix):
    l = len(prefix)
    pos = 0
    cursor = self._index
    while pos < l:
      target = prefix[pos]
      if target not in cursor:
        break
      cursor = cursor[target]
      pos += 1
    if pos == l:
      return cursor
    return None

  def magnitude(self):
    return self._largest

  def median(self):
    return self._median

  def walk(self, seek_sets):
    """Returns solutions matching `seek_sets`, ordered from high to low."""
    # TODO: Inline per findings from commit 47a736f.
    fringe = _MaxHeap()
    solutions = []
    acc = []
    fringe.push(float('inf'), (acc, self._index))
    stop_seek_pos = len(seek_sets) - 1
    while len(fringe):
      fringe_score = fringe.best_weight()
      acc, cursor = fringe.pop()
      pos = len(acc)
      seeking = seek_sets[pos]
      for c in seeking:
        if c not in cursor:
          continue
        acc.append(c)
        next_children = cursor[c]
        magnitude = next_children['max_weight']
        match_weight = next_children['match_weight']
        if match_weight:
          heapq.heappush(solutions, (-match_weight, ''.join(acc)))
        if len(next_children) > 2 and pos < stop_seek_pos:
          fringe.push(magnitude, (acc[:], next_children))
        acc.pop()
      while len(solutions) and -solutions[0][0] >= fringe_score:
        solution_weight, solution_word = heapq.heappop(solutions)
        yield solution_word, -solution_weight
    while len(solutions):
      solution_weight, solution_word = heapq.heappop(solutions)
      yield solution_word, -solution_weight

    if len(fringe._pool) > 10:
      print('WARNING')
      print('Max fringe size was: %s' % len(fringe._pool))

  def _add_to_index(self, word, weight):
    l = len(word)
    cursor = self._index
    pos = 0
    while pos < l:
      target = word[pos]
      if target not in cursor:
        break
      cursor = cursor[target]
      match_weight = cursor['match_weight']
      pos += 1
      if pos == l:
        if match_weight:
          # Duplicate addition.
          raise KeyError('%s already added' % word)
        # Exhausted input; this spells something now.
        cursor['match_weight'] = weight
        self._len += 1
        return
      elif len(cursor) <= 2:
        break  # End of Trie.
    # Add remaining characters.
    end = len(word) - 1
    for i in range(pos, len(word)):
      if i == end:
        cursor[word[i]] = {
          'max_weight': weight,
          'match_weight': weight,
        }
      else:
        new_cursor = {
          'max_weight': weight,
          'match_weight': 0,
        }
        cursor[word[i]] = new_cursor
        cursor = new_cursor
    self._len += 1

class _MaxHeap(object):
  def __init__(self):
    self._heap = []
    self._pool = []
    self._free_positions = []

  def __len__(self):
    return len(self._heap)

  def push(self, cost, o):
    if self._free_positions:
      idx = self._free_positions.pop()
      self._pool[idx] = o
    else:
      idx = len(self._pool)
      self._pool.append(o)
    heapq.heappush(self._heap, (-cost, idx))

  def best_weight(self):
    return -self._heap[0][0]

  def pop(self):
    _, idx = heapq.heappop(self._heap)
    result = self._pool[idx]
    self._pool[idx] = None
    self._free_positions.append(idx)
    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# Copyright (c) 2015 Steffen Deusch
# Licensed under the MIT license
# Beilage zu MonitorNjus, 22.07.2015 (Version 0.8.4)

import os
import sys
import imp
workingdir = os.path.dirname(os.path.realpath(__file__))
common = imp.load_source('common', workingdir+"/../common.py")

fadeinzeit = 0.8

try:
	checktime = imp.load_source('checktime', workingdir+"/../admin/checktime.py")
	import cgi
	#import cgitb; cgitb.enable()
	import getytid

	###########################

	form = cgi.FieldStorage()
	gseite = form.getvalue('seite')
	gnummer = form.getvalue('nummer')
	rows = common.getrows()
	rand = False

	###########################

	if gseite == "1":
		seite = "1"
		mseite = "Links"
	elif gseite == "2":
		seite = "2"
		mseite = "Rechts"
	else:
		raise Warning("Diese Seite existiert nicht!")

	if gnummer is None:
		x = 0
		while x < rows:
			if checktime.match(common.getinfo("VONBIS", mseite, int(common.minaktiv(mseite))+x),common.datum.now()) == True and common.getinfo("AKTIV", mseite, int(common.minaktiv(mseite))+x) == 1:
				nummer = int(common.minaktiv(mseite))+x
				break
			x += 1
	else:
		nummer = int(gnummer)

	try:
		nummer
	except:
		raise Warning("Es existiert keine aktive Seite oder ein anderer Fehler ist aufgetreten!")
		exit(0)

	###########################

	url = common.getinfo("URL", mseite, nummer)
	refresh = common.getinfo("REFRESH", mseite, nummer)

	###########################

	x = 1
	while x < rows or x == 1:
		if nummer < rows and nummer+x <= rows:
			if common.getinfo("AKTIV", mseite, nummer+x) and checktime.match(common.getinfo("VONBIS", mseite, nummer+x),common.datum.now()):
				refreshon = True
				nextnummer = nummer + x
				break
			else:
				refreshon = False
				nextnummer = nummer
		else:
			refreshon = True
			z = 0
			while z < rows:
				if checktime.match(common.getinfo("VONBIS", mseite, int(common.minaktiv(mseite))+z),common.datum.now()) == True:
					nextnummer = int(common.minaktiv(mseite))+z
					break
				else:
					nextnummer = nummer
				z += 1
		x += 1

	###########################

	if common.getinfo("REFRESHAKTIV", mseite, nummer) == 1:
		refreshon = True
	else:
		refreshon = False

	###########################

	if refreshon:
		prrefresh = '\
	<meta http-equiv="refresh" content=\"'+unicode(refresh)+'; URL=contentset.py?seite='+unicode(seite)+';nummer='+unicode(nextnummer)+'\">'
	else:
		prrefresh = None

	###########################

	typ = common.checkfiletype(url)

	if typ == "image":
		output = '

'
	elif typ == "video":
		output = '\

<video src=\''+url+'\' style="width:100%; height:auto; max-height: 100%;" autoplay="autoplay" loop="loop">Dein Browser unterstützt keine HTML5 Videos...</video>
'
	elif typ == "pdf":
		output = '\
	'
		rand = True
	elif typ == "youtube":
		output = '\
	'
		rand = True
	else:
		output = '\
	'
		rand = True

################################ HTML ################################

	print "Content-Type: text/html"
	print
	print """\
<!DOCTYPE html>

	<meta charset="UTF-8">"""
	if prrefresh is not None:
		print prrefresh
	else:
		pass
	print "\
	<title>MonitorNjus</title>"
	if typ == "video":
		style = """\
	.videocontainer
	{
		position:absolute;
		height:100%;
		width:100%;
		overflow: hidden;
		top: 0px;
	}
	.videocontainer video
	{
		min-width: 100%;
		min-height: 100%;
	}"""
	elif typ == "image":
		style = """\
	#background {
		position: absolute;
		min-height:100%;
		width: 100%;
		height: auto;
		top: 0;
		left: 0;
		background: url("""+url+""") no-repeat center center;
		background-size: contain;         /* around images */
	}"""
	else:
		style = ""
	style += """
	.fadeIn {
		opacity:0;
		-webkit-animation:fadeIn ease-in 1;
		-moz-animation:fadeIn ease-in 1;
		-o-animation:fadeIn ease-in 1;
		animation:fadeIn ease-in 1;
		-webkit-animation-fill-mode:forwards;
		-moz-animation-fill-mode:forwards;
		-o-animation-fill-mode:forwards;
		animation-fill-mode:forwards;
	}
	.fadeIn-animation {
		-webkit-animation-duration:"""+str(fadeinzeit)+"""s;
		-moz-animation-duration:"""+str(fadeinzeit)+"""s;
		-o-animation-duration:"""+str(fadeinzeit)+"""s;
		animation-duration:"""+str(fadeinzeit)+"""s;
	}
	@-webkit-keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
	@-moz-keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
	@-o-keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
	@keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }"""
	if rand:
		sys.stdout.write("""\

"""
	print "\
"
	####     # = debug     ####
	#print checktime.match(common.getinfo("VONBIS", mseite, int(common.minaktiv(mseite))),common.datum.now())
	#print nextnummer
	#print common.getinfo("VONBIS", mseite, nummer)
	#print checktime.match(common.getinfo("VONBIS", mseite, nummer),common.datum.now())
	print output
	print "\
"
	import sys
	sys.stdout.write("")
	del sys

except Exception as e:
	common.debug(e)<commit_msg>fixed bug where it didn't check if the page is active<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# Copyright (c) 2015 Steffen Deusch
# Licensed under the MIT license
# Beilage zu MonitorNjus, 22.07.2015 (Version 0.8.4)

import os
import sys
import imp
workingdir = os.path.dirname(os.path.realpath(__file__))
common = imp.load_source('common', workingdir+"/../common.py")

fadeinzeit = 0.8

try:
	checktime = imp.load_source('checktime', workingdir+"/../admin/checktime.py")
	import cgi
	#import cgitb; cgitb.enable()
	import getytid

	###########################

	form = cgi.FieldStorage()
	gseite = form.getvalue('seite')
	gnummer = form.getvalue('nummer')
	rows = common.getrows()
	rand = False

	###########################

	if gseite == "1":
		seite = "1"
		mseite = "Links"
	elif gseite == "2":
		seite = "2"
		mseite = "Rechts"
	else:
		raise Warning("Diese Seite existiert nicht!")

	if gnummer is None:
		x = 0
		while x < rows:
			if checktime.match(common.getinfo("VONBIS", mseite, int(common.minaktiv(mseite))+x),common.datum.now()) == True and common.getinfo("AKTIV", mseite, int(common.minaktiv(mseite))+x) == 1:
				nummer = int(common.minaktiv(mseite))+x
				break
			x += 1
	else:
		nummer = int(gnummer)

	try:
		nummer
	except:
		raise Warning("Es existiert keine aktive Seite oder ein anderer Fehler ist aufgetreten!")
		exit(0)

	###########################

	url = common.getinfo("URL", mseite, nummer)
	refresh = common.getinfo("REFRESH", mseite, nummer)

	###########################

	x = 1
	while x < rows or x == 1:
		if nummer < rows and nummer+x <= rows:
			if common.getinfo("AKTIV", mseite, nummer+x) and checktime.match(common.getinfo("VONBIS", mseite, nummer+x),common.datum.now()):
				refreshon = True
				nextnummer = nummer + x
				break
			else:
				refreshon = False
				nextnummer = nummer
		else:
			refreshon = True
			z = 0
			while z < rows:
				if checktime.match(common.getinfo("VONBIS", mseite, int(common.minaktiv(mseite))+z),common.datum.now()) and common.getinfo("AKTIV", mseite, int(common.minaktiv(mseite))+z):
					nextnummer = int(common.minaktiv(mseite))+z
					break
				else:
					nextnummer = nummer
				z += 1
		x += 1

	###########################

	if common.getinfo("REFRESHAKTIV", mseite, nummer) == 1:
		refreshon = True
	else:
		refreshon = False

	###########################

	if refreshon:
		prrefresh = '\
	<meta http-equiv="refresh" content=\"'+unicode(refresh)+'; URL=contentset.py?seite='+unicode(seite)+';nummer='+unicode(nextnummer)+'\">'
	else:
		prrefresh = None

	###########################

	typ = common.checkfiletype(url)

	if typ == "image":
		output = '

'
	elif typ == "video":
		output = '\

<video src=\''+url+'\' style="width:100%; height:auto; max-height: 100%;" autoplay="autoplay" loop="loop">Dein Browser unterstützt keine HTML5 Videos...</video>
'
	elif typ == "pdf":
		output = '\
	'
		rand = True
	elif typ == "youtube":
		output = '\
	'
		rand = True
	else:
		output = '\
	'
		rand = True

################################ HTML ################################

	print "Content-Type: text/html"
	print
	print """\
<!DOCTYPE html>

	<meta charset="UTF-8">"""
	if prrefresh is not None:
		print prrefresh
	else:
		pass
	print "\
	<title>MonitorNjus</title>"
	if typ == "video":
		style = """\
	.videocontainer
	{
		position:absolute;
		height:100%;
		width:100%;
		overflow: hidden;
		top: 0px;
	}
	.videocontainer video
	{
		min-width: 100%;
		min-height: 100%;
	}"""
	elif typ == "image":
		style = """\
	#background {
		position: absolute;
		min-height:100%;
		width: 100%;
		height: auto;
		top: 0;
		left: 0;
		background: url("""+url+""") no-repeat center center;
		background-size: contain;         /* around images */
	}"""
	else:
		style = ""
	style += """
	.fadeIn {
		opacity:0;
		-webkit-animation:fadeIn ease-in 1;
		-moz-animation:fadeIn ease-in 1;
		-o-animation:fadeIn ease-in 1;
		animation:fadeIn ease-in 1;
		-webkit-animation-fill-mode:forwards;
		-moz-animation-fill-mode:forwards;
		-o-animation-fill-mode:forwards;
		animation-fill-mode:forwards;
	}
	.fadeIn-animation {
		-webkit-animation-duration:"""+str(fadeinzeit)+"""s;
		-moz-animation-duration:"""+str(fadeinzeit)+"""s;
		-o-animation-duration:"""+str(fadeinzeit)+"""s;
		animation-duration:"""+str(fadeinzeit)+"""s;
	}
	@-webkit-keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
	@-moz-keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
	@-o-keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
	@keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }"""
	if rand:
		sys.stdout.write("""\

"""
	print "\
"
	####     # = debug     ####
	#print checktime.match(common.getinfo("VONBIS", mseite, int(common.minaktiv(mseite))),common.datum.now())
	#print nextnummer
	#print common.getinfo("VONBIS", mseite, nummer)
	#print checktime.match(common.getinfo("VONBIS", mseite, nummer),common.datum.now())
	print output
	print "\
"
	import sys
	sys.stdout.write("")
	del sys

except Exception as e:
	common.debug(e)<|endoftext|>


<commit_before>"""
Copyright (c) 2015 Robert Hodgen

All rights reserved.
"""

from google.appengine.ext import ndb

from datetime import datetime, timedelta, tzinfo

import utilities

from ndb_users import users

class UTC(tzinfo):
  """ Defines a `tzinfo` subclass for UTC/Universal Coordindated Time.
  All dates are stored in NDB without time zone information. This UTC class
  is needed so `datetime.isoformat()` includes timezone information. """

  def utcoffset(self, dt):
    return timedelta(0);

  def tzname(self, dt):
    return 'UTC'

  def dst(self, dt):
    return timedelta(0)

class Project(ndb.Model):
  """ Project """

  # The name/short description of this Project
  name = ndb.StringProperty(required=True)

  # The long description of this Project
  description = ndb.StringProperty(default=None)

  # The permalink (or web safe name, used in URLs) of this Project
  permalink = ndb.StringProperty(required=True)

  # The primary User's email address
  owner = ndb.StringProperty(required=True)

  # Other User's email address whom can contribute (create/end) Time Records
  contributors = ndb.StringProperty(repeated=True)

  # Other User's email address whom can view the Project and Time Records
  observers = ndb.StringProperty(repeated=True)

  # Holds the `owner`, every `contributors` and every `observers`;
  # Used for querying Projects to avoid massive OR queries...
  users = ndb.StringProperty(repeated=True)

  # The number of seconds this Project has recorded
  completed = ndb.FloatProperty(default=None)

  # Whether this Project should appear in a User's Projects list (not archived)
  active = ndb.BooleanProperty(required=True, default=True)

  # Record WHEN this record was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @property
  def created_utc(self):
    """ Return `created` with tzinfo set to `UTC`. """
    return self.created.replace(tzinfo=UTC())

  @property
  def updated_utc(self):
    """ Return `updated` with tzinfo set to `UTC`. """
    return self.updated.replace(tzinfo=UTC())

  @property
  def has_uncompleted_time_records(self):
    """ Run a query to see if this Project has uncompleted Time Records. """
    query = TimeRecord.query(TimeRecord.completed==None, ancestor=self.key)
    return (query.count(limit=1) > 0)

  @classmethod
  def create_project(cls, name):
    """ Create a new Project for the currently logged in user named `name`. """
    user = users.get_current_user()
    new_project = cls(
      name=name,
      permalink=utilities.permalinkify(name),
      owner=user.email,
      users=[user.email]
    )
    return new_project.put()

  def add_contributors(self, contributors):
    """ Add `contributors` to `self.contributors`.
    `contributors` is expected as an array of email address strings.
    Update `self.users` as needed. """
    for contributor in contributors:
      if contributor != self.owner:
        if contributor not in self.contributors:
          self.contributors.append(contributor)
        if contributor not in self.users:
          self.users.append(contributor)
        if contributor in self.observers:
          self.observers.remove(contributor)
    return self.put()

  def remove_contributors(self, contributors):
    """ Remove `contributors` from `self.contributors`.
    `contributors` is expected to be an array of email address strings.
    Update `self.users` as needed. """
    for contributor in contributors:
      if contributor in self.contributors:
        self.contributors.remove(contributor)
        if contributor in self.users and contributor != self.owner:
          self.users.remove(contributor)
    return self.put()

  def is_owner(self, email):
    """ """
    return email == self.owner

  def has_contributor(self, contributor):
    """ Returns whether `contributor` is a Contributor of this Project. """
    return contributor in self.contributors

  def has_observor(self, observer):
    """ Returns whether `observer` is an Observer of this Project. """
    return observer in self.observers

  def json_object(self):
    """ Return a dictionary representing this Project.
    Will be used for JSON requests! """
    return {
      'id': self.key.urlsafe(),
      'name': self.name,
      'owner': self.owner,
      'contributors': self.contributors,
      'observers': self.observers,
      'completed': self.completed,
      'has_uncompleted_time_records': self.has_uncompleted_time_records,
      'active': self.active,
      'created': self.created_utc.isoformat(),
      'updated': self.updated_utc.isoformat()
    }

class TimeRecord(ndb.Model):
  """ Time Record """

  tags = ndb.StringProperty(repeated=True)

  # The name/short description of this Time Record
  name = ndb.StringProperty(default=None)

  # The date/time (in UTC) this Time Record was started
  start = ndb.DateTimeProperty(required=True, auto_now_add=True)

  # Store a the User's email address who (created) this Time Record
  user = ndb.StringProperty(required=True)

  # The date/time (in UTC) this Time Record was ended
  end = ndb.DateTimeProperty(default=None)

  # The number of seconds this Time Record counted for
  completed = ndb.FloatProperty(default=None)

  # Record WHEN this record was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @classmethod
  @ndb.transactional()
  def create_time_record(cls, project_key, user_email):
    """ Create a new TimeRecord for the Project identified by `project_key`. """
    new_time_record = cls(
      parent=project_key,
      user=user_email
    )
    project_key.get().put() # Update the `updated` property
    return new_time_record.put()

  def json_object(self):
    """ Return a dictionary representing this Time Record. Will be used for
    sending information about this Time Record via JSON requests. """
    start = self.start.replace(tzinfo=UTC())
    end_string_value = None
    if self.end:
      end_string_value = self.end.replace(tzinfo=UTC()).isoformat()

    return {
      'id': self.key.urlsafe(),
      'start': start.isoformat(),
      'end': end_string_value,
      'completed': self.completed,
      'created': self.created.replace(tzinfo=UTC()).isoformat(),
      'updated': self.updated.replace(tzinfo=UTC()).isoformat(),
      'name': self.name,
      'user': self.user
    }

class Account(ndb.Model):

  pass

<commit_msg>Model update for 6c6dbbd2; will complete a Time Record<commit_after>"""
Copyright (c) 2015 Robert Hodgen

All rights reserved.
"""

from google.appengine.ext import ndb

from datetime import datetime, timedelta, tzinfo

import utilities

from ndb_users import users

class UTC(tzinfo):
  """ Defines a `tzinfo` subclass for UTC/Universal Coordindated Time.
  All dates are stored in NDB without time zone information. This UTC class
  is needed so `datetime.isoformat()` includes timezone information. """

  def utcoffset(self, dt):
    return timedelta(0);

  def tzname(self, dt):
    return 'UTC'

  def dst(self, dt):
    return timedelta(0)

class Project(ndb.Model):
  """ Project """

  # The name/short description of this Project
  name = ndb.StringProperty(required=True)

  # The long description of this Project
  description = ndb.StringProperty(default=None)

  # The permalink (or web safe name, used in URLs) of this Project
  permalink = ndb.StringProperty(required=True)

  # The primary User's email address
  owner = ndb.StringProperty(required=True)

  # Other User's email address whom can contribute (create/end) Time Records
  contributors = ndb.StringProperty(repeated=True)

  # Other User's email address whom can view the Project and Time Records
  observers = ndb.StringProperty(repeated=True)

  # Holds the `owner`, every `contributors` and every `observers`;
  # Used for querying Projects to avoid massive OR queries...
  users = ndb.StringProperty(repeated=True)

  # The number of seconds this Project has recorded
  completed = ndb.FloatProperty(default=0)

  # Whether this Project should appear in a User's Projects list (not archived)
  active = ndb.BooleanProperty(required=True, default=True)

  # Record WHEN this record was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @property
  def created_utc(self):
    """ Return `created` with tzinfo set to `UTC`. """
    return self.created.replace(tzinfo=UTC())

  @property
  def updated_utc(self):
    """ Return `updated` with tzinfo set to `UTC`. """
    return self.updated.replace(tzinfo=UTC())

  @property
  def has_uncompleted_time_records(self):
    """ Run a query to see if this Project has uncompleted Time Records. """
    query = TimeRecord.query(TimeRecord.completed==None, ancestor=self.key)
    return (query.count(limit=1) > 0)

  @classmethod
  def create_project(cls, name):
    """ Create a new Project for the currently logged in user named `name`. """
    user = users.get_current_user()
    new_project = cls(
      name=name,
      permalink=utilities.permalinkify(name),
      owner=user.email,
      users=[user.email]
    )
    return new_project.put()

  def add_contributors(self, contributors):
    """ Add `contributors` to `self.contributors`.
    `contributors` is expected as an array of email address strings.
    Update `self.users` as needed. """
    for contributor in contributors:
      if contributor != self.owner:
        if contributor not in self.contributors:
          self.contributors.append(contributor)
        if contributor not in self.users:
          self.users.append(contributor)
        if contributor in self.observers:
          self.observers.remove(contributor)
    return self.put()

  def remove_contributors(self, contributors):
    """ Remove `contributors` from `self.contributors`.
    `contributors` is expected to be an array of email address strings.
    Update `self.users` as needed. """
    for contributor in contributors:
      if contributor in self.contributors:
        self.contributors.remove(contributor)
        if contributor in self.users and contributor != self.owner:
          self.users.remove(contributor)
    return self.put()

  def is_owner(self, email):
    """ """
    return email == self.owner

  def has_contributor(self, contributor):
    """ Returns whether `contributor` is a Contributor of this Project. """
    return contributor in self.contributors

  def has_observor(self, observer):
    """ Returns whether `observer` is an Observer of this Project. """
    return observer in self.observers

  def json_object(self):
    """ Return a dictionary representing this Project.
    Will be used for JSON requests! """
    return {
      'id': self.key.urlsafe(),
      'name': self.name,
      'owner': self.owner,
      'contributors': self.contributors,
      'observers': self.observers,
      'completed': self.completed,
      'has_uncompleted_time_records': self.has_uncompleted_time_records,
      'active': self.active,
      'created': self.created_utc.isoformat(),
      'updated': self.updated_utc.isoformat()
    }

class TimeRecord(ndb.Model):
  """ Time Record """

  tags = ndb.StringProperty(repeated=True)

  # The name/short description of this Time Record
  name = ndb.StringProperty(default=None)

  # The date/time (in UTC) this Time Record was started
  start = ndb.DateTimeProperty(required=True, auto_now_add=True)

  # Store a the User's email address who (created) this Time Record
  user = ndb.StringProperty(required=True)

  # The date/time (in UTC) this Time Record was ended
  end = ndb.DateTimeProperty(default=None)

  # The number of seconds this Time Record counted for
  completed = ndb.FloatProperty(default=None)

  # Record WHEN this record was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @classmethod
  @ndb.transactional()
  def create_time_record(cls, project_key, user_email):
    """ Create a new TimeRecord for the Project identified by `project_key`. """
    new_time_record = cls(
      parent=project_key,
      user=user_email
    )
    project_key.get().put() # Update the `updated` property
    return new_time_record.put()

  def complete_time_record(self):
    """ Complete this Time Record by setting the `end` property, computing
    it's total seconds and setting this Time Record's `completed`
    property and add its seconds to it's parent Project. """

    # Only proceed if `self.completed` is None
    if not self.completed:
      # Update and record
      self.end = datetime.now()
      self.completed = (self.end - self.start).total_seconds()
      self.put()

      # Update the parent Project
      project_key = self.key.parent()
      project = project_key.get()
      project.completed += self.completed
      project.put()
    return self

  def json_object(self):
    """ Return a dictionary representing this Time Record. Will be used for
    sending information about this Time Record via JSON requests. """
    start = self.start.replace(tzinfo=UTC())
    end_string_value = None
    if self.end:
      end_string_value = self.end.replace(tzinfo=UTC()).isoformat()

    return {
      'id': self.key.urlsafe(),
      'start': start.isoformat(),
      'end': end_string_value,
      'completed': self.completed,
      'created': self.created.replace(tzinfo=UTC()).isoformat(),
      'updated': self.updated.replace(tzinfo=UTC()).isoformat(),
      'name': self.name,
      'user': self.user
    }

class Account(ndb.Model):

  pass

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import requests
from decimal import Decimal
import time
from cryptotik.common import is_sale
from cryptotik.common import (headers, ExchangeWrapper,
                              NormalizedExchangeWrapper)
from cryptotik.exceptions import (InvalidBaseCurrencyError,
                                  InvalidDelimiterError,
                                  APIError,
                                  OutdatedBaseCurrenciesError)
import hmac
import hashlib
from datetime import datetime

class Wex(ExchangeWrapper):

    def __init__(self, apikey=None, secret=None, timeout=None, proxy=None):

        if apikey:
            self.apikey = apikey.encode("utf-8")
            self.secret = secret.encode("utf-8")

        if proxy:
            assert proxy.startswith('https'), {'Error': 'Only https proxies supported.'}
        self.proxy = {'https': proxy}

        if not timeout:
            self.timeout = (8, 15)
        else:
            self.timeout = timeout

        self.api_session = requests.Session()

    public_commands = ("info", "ticker", "depth", "trades")
    private_commands = ("getInfo", "Trade", "ActiveOrders", "OrderInfo",
                        "CancelOrder", "TradeHistory", "TransHistory",
                        "WithdrawCoin", "CreateCuopon", "RedeemCuopon")

    name = 'wex'
    url = 'https://wex.nz/api/3/'
    trade_url = 'https://wex.nz/tapi/'
    delimiter = "_"
    case = "lower"
    headers = headers
    maker_fee, taker_fee = 0.002, 0.002
    base_currencies = ['eur', 'dsh', 'ltc', 'usd', 'rur', 'zec', 'btc']
    quote_order = 0

    def get_base_currencies(self):
        '''return base markets supported by this exchange.'''

        markets = [i for i in self.get_markets() if "et" not in i]  # drop tokens
        bases = list(set([i.split('_')[1].lower() for i in markets]))
        try:
            assert sorted(bases) == sorted(self.base_currencies)
        except AssertionError:
            raise OutdatedBaseCurrenciesError('Update the hardcoded base currency clist!',
                                              {'actual': bases,
                                               'hardcoded': self.base_currencies})

    def get_nonce(self):
        '''return nonce integer'''

        nonce = getattr(self, '_nonce', 0)
        if nonce:
            nonce += 1
        # If the unix time is greater though, use that instead (helps low
        # concurrency multi-threaded apps always call with the largest nonce).
        self._nonce = max(int(time.time()), nonce)
        return self._nonce

    @classmethod
    def format_pair(cls, pair):
        """format the pair argument to format understood by remote API."""

        if isinstance(pair, list):
            return pair

        pair = pair.replace("-", cls.delimiter)

        if not pair.islower():
            return pair.lower()
        else:
            return pair

    def _verify_response(self, response):
        '''verify if API responded properly and raise apropriate error.'''

        if not response.json()['success'] is 1:
            raise APIError(response.json()['error'])

    def _generate_signature(self, params):

        return hmac.new(self.secret,
                        params.encode("utf-8"),
                        hashlib.sha512).hexdigest()

    def api(self, command):
        """call remote API"""

        try:

            result = self.api_session.get(self.url + command, headers=self.headers,
                                          timeout=self.timeout, proxies=self.proxy)

            result.raise_for_status()

        except requests.exceptions.HTTPError as e:
            print(e)

        return result.json()

    def private_api(self, params):
        '''handles private api methods'''

        params["nonce"] = self.get_nonce()
        encoded_params = requests.compat.urlencode(params)

        self.headers.update({
            "Key": self.apikey,
            "Sign": self._generate_signature(encoded_params)
        })

        try:
            result = self.api_session.post(self.trade_url, data=params, headers=headers,
                                           timeout=self.timeout, proxies=self.proxy)

            result.raise_for_status()

        except requests.exceptions.HTTPError as e:
            print(e)

        self._verify_response(result)
        return result.json()

    def get_markets(self):
        '''get all pairs supported by the exchange'''

        q = self.api("info")
        return list(q['pairs'].keys())

    def get_market_ticker(self, pair):
        """return ticker for market"""

        pair = self.format_pair(pair)
        return self.api("ticker" + "/" + pair)[pair]

    def get_market_orders(self, pair, depth=100):
        """returns market order book on selected pair."""

        pair = self.format_pair(pair)

        if depth == None:
            return self.api("depth" + "/" + pair)[pair]

        if depth > 5000:
            raise ValueError("Wex API allows maximum depth of 5000 orders")

        return self.api("depth" + "/" + pair + "/?limit={0}".format(depth))[pair]

    def get_market_sell_orders(self, pair, depth=100):

        return self.get_market_orders(pair, depth)['asks']

    def get_market_buy_orders(self, pair, depth=100):

        return self.get_market_orders(pair, depth)['bids']

    def get_market_trade_history(self, pair, limit=1000):
        """get market trade history"""

        pair = self.format_pair(pair)

        if limit > 2000:
            raise APIError("Btc-e API can only return last 2000 trades.")

        if not isinstance(pair, list):
            return self.api("trades" + "/" + pair + "/?limit={0}".format(limit))[pair]

        if pair == "all":  # returns market history for all pairs with default history size.
            return self.api("trades" + "/" + "-".join(cls.get_markets() + "/?limit={0}".format(limit)))

        else:  # simply concat pairs in the list
            return self.api("trades" + "/" + "-".join(pair) + "/?limit={0}".format(limit))

    def get_market_volume(self, pair):
        '''return market volume [of last 24h]'''

        pair = self.format_pair(pair)
        smry = self.get_market_ticker(pair)

        return {pair.split(self.delimiter)[1].upper(): smry['vol'],
                pair.split(self.delimiter)[0].upper(): smry['vol_cur']}

    def get_balances(self):
        '''
        Returns information about the user’s current balance, API-key privileges,
        the number of open orders and Server Time.
        '''

        return self.private_api({"method": "getInfo"})['return']['funds']

    def get_deposit_address(self, coin=None):
        '''get deposit address'''

        return self.private_api({"method": "CoinDepositAddress", "coinName": coin})['return']

    def buy_limit(self, pair, rate, amount):
        '''submit spot buy order
        Expected result:

        * received: The amount of currency bought/sold.
        * remains: The remaining amount of currency to be bought/sold (and the initial order amount).
        * order_id: Is equal to 0 if the request was fully “matched” by the opposite orders, otherwise the ID of the executed order will be returned.
        * funds: Balance after the request.
        '''

        return self.private_api({"method": "Trade",
                                 "type": "buy",
                                 "pair": self.format_pair(pair),
                                 "amount": amount,
                                 "rate": rate
                                })['return']

    def sell_limit(self, pair, rate, amount):
        '''submit spot sell order
        Expected result:

        * received: The amount of currency bought/sold.
        * remains: The remaining amount of currency to be bought/sold (and the initial order amount).
        * order_id: Is equal to 0 if the request was fully “matched” by the opposite orders, otherwise the ID of the executed order will be returned.
        * funds: Balance after the request.
        '''

        return self.private_api({"method": "Trade",
                                 "type": "sell",
                                 "pair": self.format_pair(pair),
                                 "amount": amount,
                                 "rate": rate
                                })['return']

    def cancel_order(self, order_id):
        '''cancel order by <order_id>
        Expected result:

        * order_id: The ID of canceled order.
        * funds: Balance upon request.
        '''

        return self.private_api({"method": "CancelOrder",
                                 "order_id": order_id
                                })['return']

    def cancel_all_orders(self):
        '''cancel all active orders'''

        raise NotImplementedError

    def get_open_orders(self, pair=None):
        '''get open orders
        Expected result:

        Array key : Order ID.
        pair: The pair on which the order was created.
        type: Order type, buy/sell.
        amount: The amount of currency to be bought/sold.
        rate: Sell/Buy price.
        timestamp_created: The time when the order was created.
        '''

        if pair:
            return self.private_api({"method": "ActiveOrders", "pair": self.format_pair(pair)})['return']
        else:
            return self.private_api({"method": "ActiveOrders"})['return']

    def get_order(self, order_id):
        '''get order information'''

        return self.private_api({"method": "OrderInfo", "order_id": order_id})['return']

    def withdraw(self, coin, amount, address):
        '''withdraw cryptocurrency
        Expected result:

        tId: Transaction ID.
        amountSent: The amount sent including commission.
        funds: Balance after the request.
        '''

        return self.private_api({"method": "WithdrawCoin",
                                 "coinName": coin.upper(),
                                 "amount": amount,
                                 "address": address})['return']

    def get_transaction_history(self, since=1, until=time.time()):
        '''Returns the history of transactions.

        since: the time to start the display [unix timestamp]
        until: the time to end the display [unix timestamp]
        '''

        try:
            txn_history = self.private_api({"method": "TransHistory",
                                            "since": since,
                                            "end": int(until)})['return']
            return txn_history

        except ValueError:
            return {}

    def get_deposit_history(self, coin=None):
        '''get deposit history'''

        hist = {k: v for k, v in self.get_transaction_history().items() if  v['type'] == 1}

        if coin:
            return {k: v for k, v in hist.items() if v['currency'] == coin.upper()}

        return hist

    def get_withdraw_history(self, coin=None):
        '''get withdrawals history'''

        hist = {k: v for k, v in self.get_transaction_history().items() if  v['type'] == 2}

        if coin:
            return {k: v for k, v in hist.items() if v['currency'] == coin.upper()}

        return hist

class WexNormalized(Wex, NormalizedExchangeWrapper):

    def __init__(self, apikey=None, secret=None, timeout=None, proxy=None):
        super(WexNormalized, self).__init__(apikey, secret, timeout, proxy)

    @staticmethod
    def _tstamp_to_datetime(timestamp):
        '''convert unix timestamp to datetime'''

        return datetime.fromtimestamp(timestamp)

    @classmethod
    def format_pair(self, market_pair):
        """
        Expected input is quote - base.
        Normalize the pair inputs and
        format the pair argument to a format understood by the remote API."""

        market_pair = market_pair.lower()  # wex takes lowercase

        if "-" not in market_pair:
            raise InvalidDelimiterError('Agreed upon delimiter is "-".')

        quote, base = market_pair.split('-')

        if base not in self.base_currencies:
            raise InvalidBaseCurrencyError('''Expected input is quote-base, you have provided with {pair}'''.format(pair=market_pair))

        return quote + self.delimiter + base  # for wex quote comes first

    def get_markets(self):

        upstream = super().get_markets()

        return [i.replace('_', ('-')) for i in upstream if "et_" not in i]

    def get_market_ticker(self, market):

        ticker = super().get_market_ticker(market)

        return {
            'ask': ticker['sell'],
            'bid': ticker['buy'],
            'last': ticker['last']
        }

    def get_market_trade_history(self, market):

        upstream = super().get_market_trade_history(market)
        downstream = []

        for data in upstream:

            downstream.append({
                    'timestamp': self._tstamp_to_datetime(data['timestamp']),
                    'is_sale': is_sale(data['type']),
                    'rate': data['price'],
                    'amount': data['amount'],
                    'trade_id': data['tid']
            })

        return downstream

    def get_market_depth(self, market):
        '''return sum of all bids and asks'''

        order_book = self.get_market_orders(market)
        return {"bids": sum([Decimal(i[0]) * Decimal(i[1]) for i in order_book["bids"]]),
                "asks": sum([Decimal(i[1]) for i in order_book["asks"]])
                }

    def get_market_spread(self, market):
        '''returns market spread'''

        order_book = self.get_market_orders(market, 1)
        ask = order_book["asks"][0][0]
        bid = order_book["bids"][0][0]

        return Decimal(ask) - Decimal(bid)
<commit_msg>WexNormalized: implement get_market orders<commit_after># -*- coding: utf-8 -*-

import requests
from decimal import Decimal
import time
from cryptotik.common import is_sale
from cryptotik.common import (headers, ExchangeWrapper,
                              NormalizedExchangeWrapper)
from cryptotik.exceptions import (InvalidBaseCurrencyError,
                                  InvalidDelimiterError,
                                  APIError,
                                  OutdatedBaseCurrenciesError)
import hmac
import hashlib
from datetime import datetime

class Wex(ExchangeWrapper):

    def __init__(self, apikey=None, secret=None, timeout=None, proxy=None):

        if apikey:
            self.apikey = apikey.encode("utf-8")
            self.secret = secret.encode("utf-8")

        if proxy:
            assert proxy.startswith('https'), {'Error': 'Only https proxies supported.'}
        self.proxy = {'https': proxy}

        if not timeout:
            self.timeout = (8, 15)
        else:
            self.timeout = timeout

        self.api_session = requests.Session()

    public_commands = ("info", "ticker", "depth", "trades")
    private_commands = ("getInfo", "Trade", "ActiveOrders", "OrderInfo",
                        "CancelOrder", "TradeHistory", "TransHistory",
                        "WithdrawCoin", "CreateCuopon", "RedeemCuopon")

    name = 'wex'
    url = 'https://wex.nz/api/3/'
    trade_url = 'https://wex.nz/tapi/'
    delimiter = "_"
    case = "lower"
    headers = headers
    maker_fee, taker_fee = 0.002, 0.002
    base_currencies = ['eur', 'dsh', 'ltc', 'usd', 'rur', 'zec', 'btc']
    quote_order = 0

    def get_base_currencies(self):
        '''return base markets supported by this exchange.'''

        markets = [i for i in self.get_markets() if "et" not in i]  # drop tokens
        bases = list(set([i.split('_')[1].lower() for i in markets]))
        try:
            assert sorted(bases) == sorted(self.base_currencies)
        except AssertionError:
            raise OutdatedBaseCurrenciesError('Update the hardcoded base currency clist!',
                                              {'actual': bases,
                                               'hardcoded': self.base_currencies})

    def get_nonce(self):
        '''return nonce integer'''

        nonce = getattr(self, '_nonce', 0)
        if nonce:
            nonce += 1
        # If the unix time is greater though, use that instead (helps low
        # concurrency multi-threaded apps always call with the largest nonce).
        self._nonce = max(int(time.time()), nonce)
        return self._nonce

    @classmethod
    def format_pair(cls, pair):
        """format the pair argument to format understood by remote API."""

        if isinstance(pair, list):
            return pair

        pair = pair.replace("-", cls.delimiter)

        if not pair.islower():
            return pair.lower()
        else:
            return pair

    def _verify_response(self, response):
        '''verify if API responded properly and raise apropriate error.'''

        if not response.json()['success'] is 1:
            raise APIError(response.json()['error'])

    def _generate_signature(self, params):

        return hmac.new(self.secret,
                        params.encode("utf-8"),
                        hashlib.sha512).hexdigest()

    def api(self, command):
        """call remote API"""

        try:

            result = self.api_session.get(self.url + command, headers=self.headers,
                                          timeout=self.timeout, proxies=self.proxy)

            result.raise_for_status()

        except requests.exceptions.HTTPError as e:
            print(e)

        return result.json()

    def private_api(self, params):
        '''handles private api methods'''

        params["nonce"] = self.get_nonce()
        encoded_params = requests.compat.urlencode(params)

        self.headers.update({
            "Key": self.apikey,
            "Sign": self._generate_signature(encoded_params)
        })

        try:
            result = self.api_session.post(self.trade_url, data=params, headers=headers,
                                           timeout=self.timeout, proxies=self.proxy)

            result.raise_for_status()

        except requests.exceptions.HTTPError as e:
            print(e)

        self._verify_response(result)
        return result.json()

    def get_markets(self):
        '''get all pairs supported by the exchange'''

        q = self.api("info")
        return list(q['pairs'].keys())

    def get_market_ticker(self, pair):
        """return ticker for market"""

        pair = self.format_pair(pair)
        return self.api("ticker" + "/" + pair)[pair]

    def get_market_orders(self, pair, depth=100):
        """returns market order book on selected pair."""

        pair = self.format_pair(pair)

        if depth == None:
            return self.api("depth" + "/" + pair)[pair]

        if depth > 5000:
            raise ValueError("Wex API allows maximum depth of 5000 orders")

        return self.api("depth" + "/" + pair + "/?limit={0}".format(depth))[pair]

    def get_market_sell_orders(self, pair, depth=100):

        return self.get_market_orders(pair, depth)['asks']

    def get_market_buy_orders(self, pair, depth=100):

        return self.get_market_orders(pair, depth)['bids']

    def get_market_trade_history(self, pair, limit=1000):
        """get market trade history"""

        pair = self.format_pair(pair)

        if limit > 2000:
            raise APIError("Btc-e API can only return last 2000 trades.")

        if not isinstance(pair, list):
            return self.api("trades" + "/" + pair + "/?limit={0}".format(limit))[pair]

        if pair == "all":  # returns market history for all pairs with default history size.
            return self.api("trades" + "/" + "-".join(cls.get_markets() + "/?limit={0}".format(limit)))

        else:  # simply concat pairs in the list
            return self.api("trades" + "/" + "-".join(pair) + "/?limit={0}".format(limit))

    def get_market_volume(self, pair):
        '''return market volume [of last 24h]'''

        pair = self.format_pair(pair)
        smry = self.get_market_ticker(pair)

        return {pair.split(self.delimiter)[1].upper(): smry['vol'],
                pair.split(self.delimiter)[0].upper(): smry['vol_cur']}

    def get_balances(self):
        '''
        Returns information about the user’s current balance, API-key privileges,
        the number of open orders and Server Time.
        '''

        return self.private_api({"method": "getInfo"})['return']['funds']

    def get_deposit_address(self, coin=None):
        '''get deposit address'''

        return self.private_api({"method": "CoinDepositAddress", "coinName": coin})['return']

    def buy_limit(self, pair, rate, amount):
        '''submit spot buy order
        Expected result:

        * received: The amount of currency bought/sold.
        * remains: The remaining amount of currency to be bought/sold (and the initial order amount).
        * order_id: Is equal to 0 if the request was fully “matched” by the opposite orders, otherwise the ID of the executed order will be returned.
        * funds: Balance after the request.
        '''

        return self.private_api({"method": "Trade",
                                 "type": "buy",
                                 "pair": self.format_pair(pair),
                                 "amount": amount,
                                 "rate": rate
                                })['return']

    def sell_limit(self, pair, rate, amount):
        '''submit spot sell order
        Expected result:

        * received: The amount of currency bought/sold.
        * remains: The remaining amount of currency to be bought/sold (and the initial order amount).
        * order_id: Is equal to 0 if the request was fully “matched” by the opposite orders, otherwise the ID of the executed order will be returned.
        * funds: Balance after the request.
        '''

        return self.private_api({"method": "Trade",
                                 "type": "sell",
                                 "pair": self.format_pair(pair),
                                 "amount": amount,
                                 "rate": rate
                                })['return']

    def cancel_order(self, order_id):
        '''cancel order by <order_id>
        Expected result:

        * order_id: The ID of canceled order.
        * funds: Balance upon request.
        '''

        return self.private_api({"method": "CancelOrder",
                                 "order_id": order_id
                                })['return']

    def cancel_all_orders(self):
        '''cancel all active orders'''

        raise NotImplementedError

    def get_open_orders(self, pair=None):
        '''get open orders
        Expected result:

        Array key : Order ID.
        pair: The pair on which the order was created.
        type: Order type, buy/sell.
        amount: The amount of currency to be bought/sold.
        rate: Sell/Buy price.
        timestamp_created: The time when the order was created.
        '''

        if pair:
            return self.private_api({"method": "ActiveOrders", "pair": self.format_pair(pair)})['return']
        else:
            return self.private_api({"method": "ActiveOrders"})['return']

    def get_order(self, order_id):
        '''get order information'''

        return self.private_api({"method": "OrderInfo", "order_id": order_id})['return']

    def withdraw(self, coin, amount, address):
        '''withdraw cryptocurrency
        Expected result:

        tId: Transaction ID.
        amountSent: The amount sent including commission.
        funds: Balance after the request.
        '''

        return self.private_api({"method": "WithdrawCoin",
                                 "coinName": coin.upper(),
                                 "amount": amount,
                                 "address": address})['return']

    def get_transaction_history(self, since=1, until=time.time()):
        '''Returns the history of transactions.

        since: the time to start the display [unix timestamp]
        until: the time to end the display [unix timestamp]
        '''

        try:
            txn_history = self.private_api({"method": "TransHistory",
                                            "since": since,
                                            "end": int(until)})['return']
            return txn_history

        except ValueError:
            return {}

    def get_deposit_history(self, coin=None):
        '''get deposit history'''

        hist = {k: v for k, v in self.get_transaction_history().items() if  v['type'] == 1}

        if coin:
            return {k: v for k, v in hist.items() if v['currency'] == coin.upper()}

        return hist

    def get_withdraw_history(self, coin=None):
        '''get withdrawals history'''

        hist = {k: v for k, v in self.get_transaction_history().items() if  v['type'] == 2}

        if coin:
            return {k: v for k, v in hist.items() if v['currency'] == coin.upper()}

        return hist

class WexNormalized(Wex, NormalizedExchangeWrapper):

    def __init__(self, apikey=None, secret=None, timeout=None, proxy=None):
        super(WexNormalized, self).__init__(apikey, secret, timeout, proxy)

    @staticmethod
    def _tstamp_to_datetime(timestamp):
        '''convert unix timestamp to datetime'''

        return datetime.fromtimestamp(timestamp)

    @classmethod
    def format_pair(self, market_pair):
        """
        Expected input is quote - base.
        Normalize the pair inputs and
        format the pair argument to a format understood by the remote API."""

        market_pair = market_pair.lower()  # wex takes lowercase

        if "-" not in market_pair:
            raise InvalidDelimiterError('Agreed upon delimiter is "-".')

        quote, base = market_pair.split('-')

        if base not in self.base_currencies:
            raise InvalidBaseCurrencyError('''Expected input is quote-base, you have provided with {pair}'''.format(pair=market_pair))

        return quote + self.delimiter + base  # for wex quote comes first

    def get_markets(self):

        upstream = super().get_markets()

        return [i.replace('_', ('-')) for i in upstream if "et_" not in i]

    def get_market_ticker(self, market):

        ticker = super().get_market_ticker(market)

        return {
            'ask': ticker['sell'],
            'bid': ticker['buy'],
            'last': ticker['last']
        }

    def get_market_trade_history(self, market):

        upstream = super().get_market_trade_history(market)
        downstream = []

        for data in upstream:

            downstream.append({
                    'timestamp': self._tstamp_to_datetime(data['timestamp']),
                    'is_sale': is_sale(data['type']),
                    'rate': data['price'],
                    'amount': data['amount'],
                    'trade_id': data['tid']
            })

        return downstream

    def get_market_orders(self, market, depth=100):

        return super().get_market_orders(market, depth)

    def get_market_depth(self, market):
        '''return sum of all bids and asks'''

        order_book = self.get_market_orders(market)
        return {"bids": sum([Decimal(i[0]) * Decimal(i[1]) for i in order_book["bids"]]),
                "asks": sum([Decimal(i[1]) for i in order_book["asks"]])
                }

    def get_market_spread(self, market):
        '''returns market spread'''

        order_book = self.get_market_orders(market, 1)
        ask = order_book["asks"][0][0]
        bid = order_book["bids"][0][0]

        return Decimal(ask) - Decimal(bid)
<|endoftext|>


<commit_before>from tools.page import *
from tools.stream import extract_stream
from tools.url import UrlMgr
from tools.helper import *
from lxml import html
from lxml import etree
import re
import sys

class MovieLoads(Page):
    def __init__(self):
        self.pages_init__()

    def extract(self, url):
        url = UrlMgr({'url': url, 'log': self.log})

        try:
            media = Media(textextract(url.data, '<title>',' - Movie-Loads.NET</title>'))
        except:
            self.log.error('couldn\'t extract name, wrong url or html has changed')
            return None

        root = html.fromstring(url.data)
        part = media.createSub()
        part.name = media.name
        if root.find(".//div[@class='boxstream']") is None:
            self.log.error('No stream download found')
            return None
        for box in root.iterfind(".//div[@class='boxstream']"):
            curCol = 0

            # TODO we would need alternatives below alternatives..
            # alternative:level1 has codec/audio.. whatever information
            # alternative:level2 has the actual streems inside
            for streamBlock in box.iterfind(".//a[@rel='#overlay']"):
                alternative = part.createSub()
                alternative.name = box.find("h2").text_content()

                tmp = re.findall("language/(.*?)\.gif", etree.tostring(streamBlock))
                if len(tmp) > 0:
                    alternative.audio = tmp[0]
                tmp = re.findall("img/stream_(.*?)\.png", etree.tostring(streamBlock))
                if len(tmp) > 0:
                    alternative.hoster = tmp[0]

                streamUrl = 'http://www.movie-loads.net/'+streamBlock.get('href')
                url = UrlMgr({'url': streamUrl, 'log': self.log, 'cache_writeonly':True})
                dlUrl = 'http://www.movie-loads.net/'+textextract(url.data, '

PART 1

PART 2</li
                url = UrlMgr({'url': dlUrl, 'log': self.log, 'cache_writeonly':True})
                root = html.fromstring(url.data)
                try:
                    otherParts = root.get_element_by_id('navi_parts')
                except:
                    otherParts = False

                count = 0
                if otherParts:
                    count = 1

                # we have to reextract this part cause that site invalidates each url after it got requested once
                alternativePart = self.getAlternativePart(streamUrl, alternative, count)

                if otherParts:
                    for opart in otherParts.iterfind(".//a"):
                        if opart.get('class') == 'active':
                            continue
                        count+=1
                        streamUrl = 'http://www.movie-loads.net/'+textextract(opart.get('onclick'), "('", "')")
                        alternativePart = self.getAlternativePart(streamUrl, alternative, num)
        return media

    def getAlternativePart(self, streamUrl, alternative, num):
        part = alternative.part
        media = part.media
        alternativePart = alternative.createSub()
        url = UrlMgr({'url': streamUrl, 'log': self.log, 'cache_writeonly':True})
        streamUrl = 'http://www.movie-loads.net/'+textextract(url.data, '<iframe name="iframe" src="', '"')
        alternativePart.url = streamUrl
        if num:
            alternativePart.num = num
        self.setPinfo(alternativePart)
        return alternativePart

urlPart = 'movie-loads' # this part will be matched in __init__ to create following class
classRef = MovieLoads
<commit_msg>allpages for movieloads<commit_after>from tools.page import *
from tools.stream import extract_stream
from tools.url import UrlMgr
from tools.helper import *
from lxml import html
from lxml import etree
import re
import sys

class MovieLoads(Page):
    def __init__(self):
        self.pages_init__()

    def getAllPages(self):
        allPages = []
        url = UrlMgr({'url': 'http://www.movie-loads.net/?movies', 'log': self.log})
        root = html.fromstring(url.data)
        lastPageContainer = root.get_element_by_id('navi_sub2')
        # get the last a link in this container
        lastPageA = lastPageContainer.findall(".//a")[-1]
        lastPage = textextract(lastPageA.get('href'), 'page=', '')
        self.log.info("Get all movies from "+lastPage+" pages.")
        for pageNum in range(1, int(lastPage)+1):
            self.log.info("page "+str(pageNum))
            url = UrlMgr({'url': 'http://www.movie-loads.net/?movies&page='+str(pageNum), 'log': self.log})
            root = html.fromstring(url.data)
            for movie in root.iterfind(".//div[@class='movie']"):
                mediaId = textextract(movie.find(".//a").get('href'), 'media=', '')
                mediaUrl = 'http://www.movie-loads.net/?media='+mediaId
                media = self.extract(mediaUrl)
                media.img = 'http://www.movie-loads.net/cover/tn/'+mediaId+'.jpg'
                self.log.info("finished page '"+media.name+"'")
                allPages.append(media)
        return allPages

    def extract(self, url):
        url = UrlMgr({'url': url, 'log': self.log})

        try:
            media = Media(textextract(url.data, '<title>',' - Movie-Loads.NET</title>'))
        except:
            self.log.error('couldn\'t extract name, wrong url or html has changed')
            return None

        root = html.fromstring(url.data)
        part = media.createSub()
        part.name = media.name
        if root.find(".//div[@class='boxstream']") is None:
            self.log.error('No stream download found')
            return None
        for box in root.iterfind(".//div[@class='boxstream']"):
            curCol = 0

            # TODO we would need alternatives below alternatives..
            # alternative:level1 has codec/audio.. whatever information
            # alternative:level2 has the actual streems inside
            for streamBlock in box.iterfind(".//a[@rel='#overlay']"):
                alternative = part.createSub()
                alternative.name = box.find("h2").text_content()

                tmp = re.findall("language/(.*?)\.gif", etree.tostring(streamBlock))
                if len(tmp) > 0:
                    alternative.audio = tmp[0]
                tmp = re.findall("img/stream_(.*?)\.png", etree.tostring(streamBlock))
                if len(tmp) > 0:
                    alternative.hoster = tmp[0]

                streamUrl = 'http://www.movie-loads.net/'+streamBlock.get('href')
                url = UrlMgr({'url': streamUrl, 'log': self.log, 'cache_writeonly':True})
                dlUrl = 'http://www.movie-loads.net/'+textextract(url.data, '

PART 1

PART 2</li
                url = UrlMgr({'url': dlUrl, 'log': self.log, 'cache_writeonly':True})
                root = html.fromstring(url.data)
                try:
                    otherParts = root.get_element_by_id('navi_parts')
                except:
                    otherParts = False

                count = 0
                if otherParts:
                    count = 1

                # we have to reextract this part cause that site invalidates each url after it got requested once
                alternativePart = self.getAlternativePart(streamUrl, alternative, count)

                if otherParts:
                    for opart in otherParts.iterfind(".//a"):
                        if opart.get('class') == 'active':
                            continue
                        count+=1
                        streamUrl = 'http://www.movie-loads.net/'+textextract(opart.get('onclick'), "('", "')")
                        alternativePart = self.getAlternativePart(streamUrl, alternative, num)
        return media

    def getAlternativePart(self, streamUrl, alternative, num):
        part = alternative.part
        media = part.media
        alternativePart = alternative.createSub()
        url = UrlMgr({'url': streamUrl, 'log': self.log, 'cache_writeonly':True})
        streamUrl = 'http://www.movie-loads.net/'+textextract(url.data, '<iframe name="iframe" src="', '"')
        alternativePart.url = streamUrl
        if num:
            alternativePart.num = num
        self.setPinfo(alternativePart)
        return alternativePart

urlPart = 'movie-loads' # this part will be matched in __init__ to create following class
classRef = MovieLoads
<|endoftext|>