<commit_before>def Pythagoras_tree(m = 0.8, n = 12):
    # Compute Pythagoras_tree
    # The Pythagoras Tree is a plane fractal constructed from squares.
    # It is named after Pythagoras  because each triple of touching squares
    # encloses a right triangle, in a configuration traditionally used to
    # depict the Pythagorean theorem.
    # http://en.wikipedia.org/wiki/Pythagoras_tree
    #
    #       All these arguments are optional: the function can run with
    #       argument.
    # Output :
    #       - Matrix M: Pythagoras tree is stored in a matrix M.
    #         This matrix has 5 columns.
    #         Each row corresponds to the coordinate of each square of the tree
    #         The two first columns give the bottom-left position of each
    #         square. The third column corresponds to the orientation angle of
    #         each square. The fourth column gives the size of each square. The
    #         fifth column specifies the level of recursion of each square.
    #         The first row corresponds to the root of the tree. It is always
    #         the same
    #         M[0,:] = [0 -1 0 1 1];
    #         The leaf located at row i will give 2 leaves located at 2*i and
    #         2*i+1.

    # Check inputs
    from math import pi, atan2, sqrt
    import numpy as np
    if m <= 0:
        raise Exception('Length of m has to be greater than zero')
    if int(n)!=float(n):
        raise Exception('The number of level has to be integer')

    ## Compute constants
    d      = sqrt(1+m**2)                                   #
    c1     = 1/d                                            # Normalized length 1
    c2     = m/d                                            # Normalized length 2
    T      = np.array([[0,1/(1+m**2)],[1,1+m/(1+m**2)]])    # Translation pattern
    alpha1 = atan2(m,1)                                     # Defines the first rotation angle
    alpha2 = alpha1-pi/2                                    # Defines the second rotation angle
    pi2    = 2*pi                                           # Defines pi2
    nEle   = 2**(n+1)-1                                     # Number of elements (square)
    M      = np.zeros((nEle,5))                             # Matrice containing the tree
    M[0,:] = [0,-1,0,1,1]                                   # Initialization of the tree

    # Compute the level of each square contained in the resulting matrix
    Offset = 0
    for i in range(n+1):
        tmp = 2**i
        M[Offset : Offset + tmp, 4] = i;
        Offset = Offset + tmp

    def mat_rot(x):
        import numpy as np
        c = np.cos(x)
        s = np.sin(x)
        return np.array([[c,-s],[s,c]])
    # Compute the position and size of each square wrt its parent
    for i in range(1,nEle,2):
        j          = (i+1)//2-1
        mT         = M[j,3] * np.dot(mat_rot(M[j,2]), T )
        Tx         = mT[0,:] + M[j,0]
        Ty         = mT[1,:] + M[j,1]
        theta1     = (M[j,2]+alpha1)%pi2
        theta2     = (M[j,2]+alpha2)%pi2
        M[i  ,0:4] = [Tx[0],Ty[0],theta1,M[j,3]*c1]
        M[i+1,0:4] = [Tx[1],Ty[1],theta2,M[j,3]*c2]
    return M

def Pythagor_tree_write2svg(m = 0.8, n = 12, colormap = 'summer', M = []):
    # A svg file giving a vectorial display of the tree. The name of
    # file is generated from the parameter m,n,colormap. The file is
    # stored in the current folder.
    #
    # Determine the bounding box of the tree with an offset
    # Display_metadata = false;
    from math import pi, atan2, sqrt, ceil
    import numpy as np
    import matplotlib
    from matplotlib import cm
    Display_metadata = True

    nEle    = M.shape[0]
    r2      = sqrt(2)
    LOffset = M[nEle-1,3] + 0.1
    min_x   = np.min(M[:,0]-r2*M[:,3]) - LOffset
    max_x   = np.max(M[:,0]+r2*M[:,3]) + LOffset
    min_y   = np.min(M[:,1]          ) - LOffset
    max_y   = np.max(M[:,1]+r2*M[:,3]) + LOffset

    # Compute the color of tree
    ColorM = cm.get_cmap(colormap)
    co   = 100;
    Wfig = ceil(co*(max_x-min_x))
    Hfig = ceil(co*(max_y-min_y))
    filename = 'Pythagoras_tree_'+str(m).replace('.','_')+'__'+str(n)+'__'+colormap+'.svg'
    fid  = open(filename, 'wt');
    fid.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n')
    if not Display_metadata:
        fid.write('<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"\n')
        fid.write('  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">\n')
    fid.write('<svg width="{0}" height="{1}" version="1.1"\n'.format(Wfig,Hfig))
    # fid.write(['<svg width="12cm" height="4cm" version="1.1"\n']); % Wfig,

    # fid.write(['<svg width="15cm" height="10cm" '...
    #              'viewBox="0 0 %d %d" version="1.1"\n'],...
    #              Wfig,Hfig);
    if Display_metadata:
        fid.write('\txmlns:dc="http://purl.org/dc/elements/1.1/"\n')
        fid.write('\txmlns:cc="http://creativecommons.org/ns#"\n')
        fid.write('\txmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"\n')
    fid.write('\txmlns:svg="http://www.w3.org/2000/svg"\n')
    fid.write('\txmlns="http://www.w3.org/2000/svg"\n')
    fid.write('\txmlns:xlink="http://www.w3.org/1999/xlink">\n')

    if Display_metadata:
        fid.write('\t<title>Pythagoras tree</title>\n')
        fid.write('\t<metadata>\n')
        fid.write('\t\t<rdf:RDF>\n')
        fid.write('\t\t\t<cc:Work\n')
        fid.write('\t\t\t\trdf:about="">\n')
        fid.write('\t\t\t\t<dc:format>image/svg+xml</dc:format>\n')
        fid.write('\t\t\t\t<dc:type\n')
        fid.write('\t\t\t\t\trdf:resource="http://purl.org/dc/dcmitype/StillImage" />\n')
        fid.write('\t\t\t\t<dc:title>Pythagoras tree</dc:title>\n')
        fid.write('\t\t\t\t<dc:creator>\n')
        fid.write('\t\t\t\t\t<cc:Agent>\n')
        fid.write('\t\t\t\t\t\t<dc:title>Guillaume Jacquenot</dc:title>\n')
        fid.write('\t\t\t\t\t</cc:Agent>\n')
        fid.write('\t\t\t\t</dc:creator>\n')
        fid.write('\t\t\t\t<cc:license\n')
        fid.write('\t\t\t\t\t\trdf:resource="http://creativecommons.org/licenses/by-nc-sa/3.0/" />\n')
        fid.write('\t\t\t</cc:Work>\n')
        fid.write('\t\t\t<cc:License\n')
        fid.write('\t\t\t\trdf:about="http://creativecommons.org/licenses/by-nc-sa/3.0/">\n')
        fid.write('\t\t\t\t<cc:permits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Reproduction" />\n')
        fid.write('\t\t\t\t<cc:permits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Reproduction" />\n')
        fid.write('\t\t\t\t<cc:permits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Distribution" />\n')
        fid.write('\t\t\t\t<cc:requires\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Notice" />\n')
        fid.write('\t\t\t\t<cc:requires\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Attribution" />\n')
        fid.write('\t\t\t\t<cc:prohibits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#CommercialUse" />\n')
        fid.write('\t\t\t\t<cc:permits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#DerivativeWorks" />\n')
        fid.write('\t\t\t\t<cc:requires\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#ShareAlike" />\n')
        fid.write('\t\t\t</cc:License>\n')
        fid.write('\t\t</rdf:RDF>\n')
        fid.write('\t</metadata>\n')
    fid.write('\t<defs>\n')
    fid.write('\t\t<rect width="{0}" height="{1}" \n'.format(co,co))
    fid.write('\t\t\tx="0" y="0"\n')
    fid.write('\t\t\tstyle="fill-opacity:1;stroke:#00d900;stroke-opacity:1"\n')
    fid.write('\t\t\tid="squa"\n')
    fid.write('\t\t/>\n')
    fid.write('\t</defs>\n')
    fid.write('\t<g transform="translate({0} {1}) rotate(180) " >\n'.format(round(co*max_x),round(co*max_y)))
    for i in range(n+1):
        fid.write('\t\t<g style="fill:{0};" >\n'.format(matplotlib.colors.rgb2hex(ColorM(1.0-i/(n+1)))))
        Offset = 2**i-1
        for j in range(2**i):
            k = j + Offset
            fid.write('\t\t\t<use xlink:href="#squa" transform="translate({0:+010.5f} {1:+010.5f}) rotate({2:+010.5f}) scale({3:8.6f})" />\n'.format(co*M[k,0],co*M[k,1],M[k,2]*180/pi,M[k,3]))
        fid.write('\t\t</g>\n')
    fid.write('\t</g>\n')
    fid.write('</svg>\n')
    fid.close()

#def Pythagor_tree_plot(D, colormap = 'summer'):
#    from matplotlib import cm
#    ColorM = cm.get_cmap(colormap)
#    H = figure('color','w');
#    hold on
#    axis equal
#    axis off
#    def mat_rot(x):
#       import numpy as np
#       c = np.cos(x)
#       s = np.sin(x)
#       return np.array([[c,-s],[s,c]])
#    for i in range(M.shape[0]):
#        cx    = D[i,0]
#        cy    = D[i,1]
#        theta = D[i,2]
#        si    = D[i,3]
#        M     = mat_rot(theta)
#        x     = si*np.array([0,1,1,0,0])
#        y     = si*np.array([0,0,1,1,0])
#        pts   = np.dot(M,np.array([[x],[y]]))
#        fill(cx+pts[0,:],cy+pts[1,:],ColorM(D(i,5)+1,:));
#        # plot(cx+pts(1,1:2),cy+pts(2,1:2),'r');

def iscolormap(cmap):
    # This function returns true if 'cmap' is a valid colormap
    LCmap = [\
        'autumn',
        'bone',
        'colorcube',
        'cool',
        'copper',
        'flag',
        'gray',
        'hot',
        'hsv',
        'jet',
        'lines',
        'pink',
        'prism',
        'spring',
        'summer',
        'white',
        'winter']
    return cmap in LCmap

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description = 'This script creates a SVG image of a Pythagoras tree, a plane fractal constructed from squares.')
    parser.add_argument('m', type = float, default = 1.0,
                       help = 'm ( m > 0 ) is the relative length of one of the side right-angled triangle. '+
                              'The second side of the right-angle is taken to be one. '+
                              'To have a symmetric tree, m has to be 1.')
    parser.add_argument('n', type = int, default = 10,
                       help='n is the level of recursion. The number of elements of tree is equal '+
                            'to 2**(n+1)-1. A reasonable number for n is 10.')
    parser.add_argument('-c','--colormap', type = str, default = 'summer',
                       help='Matplotlib colormap used to generate color of the different levels of the tree.')
    args = parser.parse_args()

    M = Pythagoras_tree(m = args.m, n = args.n)
    # Display the tree
    # Pythagor_tree_plot(M,n);
    # Write results to an SVG file
    Pythagor_tree_write2svg(args.m, args.n, args.colormap, M)

<commit_msg>[master] Added an export function in the Python script<commit_after>def Pythagoras_tree(m = 0.8, n = 12):
    # Compute Pythagoras_tree
    # The Pythagoras Tree is a plane fractal constructed from squares.
    # It is named after Pythagoras  because each triple of touching squares
    # encloses a right triangle, in a configuration traditionally used to
    # depict the Pythagorean theorem.
    # http://en.wikipedia.org/wiki/Pythagoras_tree
    #
    #       All these arguments are optional: the function can run with
    #       argument.
    # Output :
    #       - Matrix M: Pythagoras tree is stored in a matrix M.
    #         This matrix has 5 columns.
    #         Each row corresponds to the coordinate of each square of the tree
    #         The two first columns give the bottom-left position of each
    #         square. The third column corresponds to the orientation angle of
    #         each square. The fourth column gives the size of each square. The
    #         fifth column specifies the level of recursion of each square.
    #         The first row corresponds to the root of the tree. It is always
    #         the same
    #         M[0,:] = [0 -1 0 1 1];
    #         The leaf located at row i will give 2 leaves located at 2*i and
    #         2*i+1.

    # Check inputs
    from math import pi, atan2, sqrt
    import numpy as np
    if m <= 0:
        raise Exception('Length of m has to be greater than zero')
    if int(n)!=float(n):
        raise Exception('The number of level has to be integer')

    ## Compute constants
    d      = sqrt(1+m**2)                                   #
    c1     = 1/d                                            # Normalized length 1
    c2     = m/d                                            # Normalized length 2
    T      = np.array([[0,1/(1+m**2)],[1,1+m/(1+m**2)]])    # Translation pattern
    alpha1 = atan2(m,1)                                     # Defines the first rotation angle
    alpha2 = alpha1-pi/2                                    # Defines the second rotation angle
    pi2    = 2*pi                                           # Defines pi2
    nEle   = 2**(n+1)-1                                     # Number of elements (square)
    M      = np.zeros((nEle,5))                             # Matrice containing the tree
    M[0,:] = [0,-1,0,1,1]                                   # Initialization of the tree

    # Compute the level of each square contained in the resulting matrix
    Offset = 0
    for i in range(n+1):
        tmp = 2**i
        M[Offset : Offset + tmp, 4] = i;
        Offset = Offset + tmp

    def mat_rot(x):
        import numpy as np
        c = np.cos(x)
        s = np.sin(x)
        return np.array([[c,-s],[s,c]])
    # Compute the position and size of each square wrt its parent
    for i in range(1,nEle,2):
        j          = (i+1)//2-1
        mT         = M[j,3] * np.dot(mat_rot(M[j,2]), T )
        Tx         = mT[0,:] + M[j,0]
        Ty         = mT[1,:] + M[j,1]
        theta1     = (M[j,2]+alpha1)%pi2
        theta2     = (M[j,2]+alpha2)%pi2
        M[i  ,0:4] = [Tx[0],Ty[0],theta1,M[j,3]*c1]
        M[i+1,0:4] = [Tx[1],Ty[1],theta2,M[j,3]*c2]
    return M

def Pythagor_tree_write2svg(m = 0.8, n = 12, colormap = 'summer', M = []):
    # A svg file giving a vectorial display of the tree. The name of
    # file is generated from the parameter m,n,colormap. The file is
    # stored in the current folder.
    #
    # Determine the bounding box of the tree with an offset
    # Display_metadata = false;
    from math import pi, atan2, sqrt, ceil
    import numpy as np
    import matplotlib
    from matplotlib import cm
    Display_metadata = True

    nEle    = M.shape[0]
    r2      = sqrt(2)
    LOffset = M[nEle-1,3] + 0.1
    min_x   = np.min(M[:,0]-r2*M[:,3]) - LOffset
    max_x   = np.max(M[:,0]+r2*M[:,3]) + LOffset
    min_y   = np.min(M[:,1]          ) - LOffset
    max_y   = np.max(M[:,1]+r2*M[:,3]) + LOffset

    # Compute the color of tree
    ColorM = cm.get_cmap(colormap)
    co   = 100;
    Wfig = ceil(co*(max_x-min_x))
    Hfig = ceil(co*(max_y-min_y))
    filename = 'Pythagoras_tree_'+str(m).replace('.','_')+'__'+str(n)+'__'+colormap+'.svg'
    fid  = open(filename, 'wt');
    fid.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n')
    if not Display_metadata:
        fid.write('<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"\n')
        fid.write('  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">\n')
    fid.write('<svg width="{0}" height="{1}" version="1.1"\n'.format(Wfig,Hfig))
    # fid.write(['<svg width="12cm" height="4cm" version="1.1"\n']); % Wfig,

    # fid.write(['<svg width="15cm" height="10cm" '...
    #              'viewBox="0 0 %d %d" version="1.1"\n'],...
    #              Wfig,Hfig);
    if Display_metadata:
        fid.write('\txmlns:dc="http://purl.org/dc/elements/1.1/"\n')
        fid.write('\txmlns:cc="http://creativecommons.org/ns#"\n')
        fid.write('\txmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"\n')
    fid.write('\txmlns:svg="http://www.w3.org/2000/svg"\n')
    fid.write('\txmlns="http://www.w3.org/2000/svg"\n')
    fid.write('\txmlns:xlink="http://www.w3.org/1999/xlink">\n')

    if Display_metadata:
        fid.write('\t<title>Pythagoras tree</title>\n')
        fid.write('\t<metadata>\n')
        fid.write('\t\t<rdf:RDF>\n')
        fid.write('\t\t\t<cc:Work\n')
        fid.write('\t\t\t\trdf:about="">\n')
        fid.write('\t\t\t\t<dc:format>image/svg+xml</dc:format>\n')
        fid.write('\t\t\t\t<dc:type\n')
        fid.write('\t\t\t\t\trdf:resource="http://purl.org/dc/dcmitype/StillImage" />\n')
        fid.write('\t\t\t\t<dc:title>Pythagoras tree</dc:title>\n')
        fid.write('\t\t\t\t<dc:creator>\n')
        fid.write('\t\t\t\t\t<cc:Agent>\n')
        fid.write('\t\t\t\t\t\t<dc:title>Guillaume Jacquenot</dc:title>\n')
        fid.write('\t\t\t\t\t</cc:Agent>\n')
        fid.write('\t\t\t\t</dc:creator>\n')
        fid.write('\t\t\t\t<cc:license\n')
        fid.write('\t\t\t\t\t\trdf:resource="http://creativecommons.org/licenses/by-nc-sa/3.0/" />\n')
        fid.write('\t\t\t</cc:Work>\n')
        fid.write('\t\t\t<cc:License\n')
        fid.write('\t\t\t\trdf:about="http://creativecommons.org/licenses/by-nc-sa/3.0/">\n')
        fid.write('\t\t\t\t<cc:permits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Reproduction" />\n')
        fid.write('\t\t\t\t<cc:permits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Reproduction" />\n')
        fid.write('\t\t\t\t<cc:permits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Distribution" />\n')
        fid.write('\t\t\t\t<cc:requires\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Notice" />\n')
        fid.write('\t\t\t\t<cc:requires\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#Attribution" />\n')
        fid.write('\t\t\t\t<cc:prohibits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#CommercialUse" />\n')
        fid.write('\t\t\t\t<cc:permits\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#DerivativeWorks" />\n')
        fid.write('\t\t\t\t<cc:requires\n')
        fid.write('\t\t\t\t\trdf:resource="http://creativecommons.org/ns#ShareAlike" />\n')
        fid.write('\t\t\t</cc:License>\n')
        fid.write('\t\t</rdf:RDF>\n')
        fid.write('\t</metadata>\n')
    fid.write('\t<defs>\n')
    fid.write('\t\t<rect width="{0}" height="{1}" \n'.format(co,co))
    fid.write('\t\t\tx="0" y="0"\n')
    fid.write('\t\t\tstyle="fill-opacity:1;stroke:#00d900;stroke-opacity:1"\n')
    fid.write('\t\t\tid="squa"\n')
    fid.write('\t\t/>\n')
    fid.write('\t</defs>\n')
    fid.write('\t<g transform="translate({0} {1}) rotate(180) " >\n'.format(round(co*max_x),round(co*max_y)))
    for i in range(n+1):
        fid.write('\t\t<g style="fill:{0};" >\n'.format(matplotlib.colors.rgb2hex(ColorM(1.0-i/(n+1)))))
        Offset = 2**i-1
        for j in range(2**i):
            k = j + Offset
            fid.write('\t\t\t<use xlink:href="#squa" transform="translate({0:+010.5f} {1:+010.5f}) rotate({2:+010.5f}) scale({3:8.6f})" />\n'.format(co*M[k,0],co*M[k,1],M[k,2]*180/pi,M[k,3]))
        fid.write('\t\t</g>\n')
    fid.write('\t</g>\n')
    fid.write('</svg>\n')
    fid.close()

def Pythagor_tree_plot(M, colormap = 'summer', outputFilename = 'lm.png'):
    from math import pi
    from matplotlib import cm
    from matplotlib import patches
    import matplotlib.pyplot as plt
    ColorM = cm.get_cmap(colormap)
    fig, ax = plt.subplots()
    for i in range(M.shape[0]):
        cx    = M[i,0]
        cy    = M[i,1]
        theta = M[i,2]
        si    = M[i,3]
        rect = patches.Rectangle([cx,cy], si, si, angle = theta * 180.0/pi, ec="none", color = ColorM(1.0-i/(M[-1,4]+1)))
        ax.add_patch(rect)
    plt.xlim([-4, 4])
    plt.ylim([-1.5, 3.5])
    # plt.gca().relim()
    plt.gca().set_aspect('equal', adjustable='box')
    plt.axis('off')
    fig.savefig(outputFilename, bbox_inches='tight')

def iscolormap(cmap):
    # This function returns true if 'cmap' is a valid colormap
    LCmap = [\
        'autumn',
        'bone',
        'colorcube',
        'cool',
        'copper',
        'flag',
        'gray',
        'hot',
        'hsv',
        'jet',
        'lines',
        'pink',
        'prism',
        'spring',
        'summer',
        'white',
        'winter']
    return cmap in LCmap

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description = 'This script creates a SVG image of a Pythagoras tree, a plane fractal constructed from squares.')
    parser.add_argument('m', type = float, default = 1.0,
                       help = 'm ( m > 0 ) is the relative length of one of the side right-angled triangle. '+
                              'The second side of the right-angle is taken to be one. '+
                              'To have a symmetric tree, m has to be 1.')
    parser.add_argument('n', type = int, default = 10,
                       help='n is the level of recursion. The number of elements of tree is equal '+
                            'to 2**(n+1)-1. A reasonable number for n is 10.')
    parser.add_argument('-c','--colormap', type = str, default = 'summer',
                       help='Matplotlib colormap used to generate color of the different levels of the tree.')
    args = parser.parse_args()

    M = Pythagoras_tree(m = args.m, n = args.n)
    # Display the tree
    # Pythagor_tree_plot(M,n);
    # Write results to an SVG file
    Pythagor_tree_write2svg(args.m, args.n, args.colormap, M)

<|endoftext|>


<commit_before>#import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import numpy.random as rng
import pandas.io.data as web
import numpy as np
import os
import pandas as pd

# we modify this data organizing slightly to get two symbols
def get_prices(symbol):
    start, end = '2007-05-02', '2016-04-11'
    data = web.DataReader(symbol, 'yahoo', start, end)
    data=pd.DataFrame(data)
    prices=data['Adj Close']
    prices=prices.astype(float)
    return prices

def get_returns(prices):
        return ((prices-prices.shift(-1))/prices)[:-1]

def get_data(list):
    l = []
    for symbol in list:
        rets = get_returns(get_prices(symbol))
        l.append(rets)
    return np.array(l).T

def sort_data(rets):
    ins = []
    outs = []
    for i in range(len(rets)-100):
        ins.append(rets[i:i+100].tolist())
        outs.append(rets[i+100])
    return np.array(ins), np.array(outs)

print('loading data')
symbol_list = ['C', 'GS']
def data_loader_1(symbol_list):
    if os.path.exists('rets.pkl'):
        rets = pd.read_pickle('rets.pkl')
    else:
        rets = get_data(symbol_list)
        pd.to_pickle(rets, 'rets.pkl')
    ins, outs = sort_data(rets)
    ins = ins.transpose([0,2,1]).reshape([-1, len(symbol_list) * 100])
    div = int(.8 * ins.shape[0])
    train_ins, train_outs = ins[:div], outs[:div]
    test_ins, test_outs = ins[div:], outs[div:]

    #normalize inputs
    train_ins, test_ins = train_ins/np.std(ins), test_ins/np.std(ins)
    return train_ins, test_ins, train_outs, test_outs
train_ins, test_ins, train_outs, test_outs = data_loader_1(symbol_list)
print('loaded data')

sess = tf.Session()

positions = tf.constant([-1,0,1]) #long, neutral or short
num_positions = 3
num_symbols = len(symbol_list)
num_samples = 50

n_input = num_symbols * 100
n_hidden_1 = 10 # 1st layer number of features
n_hidden_2 = 10 # 2nd layer number of features
n_classes = num_positions * num_symbols # MNIST total classes (0-9 digits)

# define placeholders
x = tf.placeholder(tf.float32, [None, num_symbols * 100])
y_ = tf.placeholder(tf.float32, [None,  num_symbols])

weights = {
    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
}
biases = {
    'b1': tf.Variable(tf.random_normal([n_hidden_1])),
    'b2': tf.Variable(tf.random_normal([n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_classes]))
}

def multilayer_perceptron(x, weights, biases):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.relu(layer_1)
    # Hidden layer with RELU activation
    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
    layer_2 = tf.nn.relu(layer_2)
    # Output layer with linear activation
    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
    return out_layer

# Construct model
y = multilayer_perceptron(x, weights, biases)

# loop through symbol, taking the columns for each symbol's bucket together
pos = {}
sample_n = {}
sample_mask = {}
symbol_returns = {}
relevant_target_column = {}
for i in range(num_symbols):
    # isolate the buckets relevant to the symbol and get a softmax as well
    symbol_probs = y[:,i*num_positions:(i+1)*num_positions]
    symbol_probs_softmax = tf.nn.softmax(symbol_probs) # softmax[i, j] = exp(logits[i, j]) / sum(exp(logits[i]))
    # sample probability to chose our policy's action
    sample = tf.multinomial(tf.log(symbol_probs_softmax), num_samples)
    for sample_iter in range(num_samples):
        sample_n[i*num_samples + sample_iter] = sample[:,sample_iter]
        pos[i*num_samples + sample_iter] = tf.reshape(sample_n[i*num_samples + sample_iter], [-1]) - 1
        symbol_returns[i*num_samples + sample_iter] = tf.mul(
                                                            tf.cast(pos[i*num_samples + sample_iter], np.float32),
                                                             y_[:,i])

        sample_mask[i*num_samples + sample_iter] = tf.cast(tf.reshape(tf.one_hot(sample_n[i*num_samples + sample_iter], 3), [-1,3]), np.float32)
        relevant_target_column[i*num_samples + sample_iter] = tf.reduce_sum(
                                                    symbol_probs * sample_mask[i*num_samples + sample_iter],1)

daily_returns_by_symbol_ = tf.concat(1, [tf.reshape(t, [-1,1]) for t in symbol_returns.values()])
daily_returns_by_symbol = tf.transpose(tf.reshape(daily_returns_by_symbol_, [-1,2,num_samples]), [0,2,1]) #[?,5,2]
daily_returns = tf.reduce_mean(daily_returns_by_symbol, 2) # [?,5]

total_return = tf.reduce_prod(daily_returns+1, 0)
z = tf.ones_like(total_return) * -1
total_return = tf.add(total_return, z)

ann_vol = tf.mul(
    tf.sqrt(tf.reduce_mean(tf.pow((daily_returns - tf.reduce_mean(daily_returns, 0)),2),0)) ,
    np.sqrt(252)
    )
sharpe = tf.div(total_return, ann_vol)
#Maybe metric slicing later
#segment_ids = tf.ones_like(daily_returns[:,0])
#partial_prod = tf.segment_prod(daily_returns+1, segment_ids)

training_target_cols = tf.concat(1, [tf.reshape(t, [-1,1]) for t in relevant_target_column.values()])
ones = tf.ones_like(training_target_cols)
gradient_ = tf.nn.sigmoid_cross_entropy_with_logits(training_target_cols, ones)

gradient = tf.transpose(tf.reshape(gradient_, [-1,2,num_samples]), [0,2,1]) #[?,5,2]

#cost = tf.mul(gradient , daily_returns_by_symbol_reshaped)
#cost = tf.mul(gradient , tf.expand_dims(daily_returns, -1))
cost = tf.mul(gradient , tf.expand_dims(total_return, -1))
#cost = tf.mul(gradient , tf.expand_dims(sharpe, -1))

optimizer = tf.train.GradientDescentOptimizer(0.0001).minimize(cost)
costfn = tf.reduce_mean(cost)

def run_training_and_return_test_results():
    print('START TRAINING_______________________________')
    # initialize variables to random values
    init = tf.initialize_all_variables()
    sess.run(init)
    # run optimizer on entire training data set many times
    train_size = train_ins.shape[0]
    for epoch in range(2000):
        start = rng.randint(train_size-50)
        batch_size = rng.randint(2,25)
        end = min(train_size, start+batch_size)

        sess.run(optimizer, feed_dict={x: train_ins[start:end], y_: train_outs[start:end]})#.reshape(1,-1).T})
        # every 1000 iterations record progress
        if (epoch+1)%100== 0:
            t,s, c = sess.run([ total_return, sharpe, costfn], feed_dict={x: train_ins, y_: train_outs})#.reshape(1,-1).T})
            t = np.mean(t)
            s = np.mean(s)
            print("Epoch:", '%04d' % (epoch+1), "cost=",c, "total return=", "{:.9f}".format(t),
                 "sharpe=", "{:.9f}".format(s))
            #print(t)

    print('DONE TRAINING _______________________________')

    d_tr, t_tr = sess.run([daily_returns, total_return], feed_dict={x: train_ins, y_: train_outs})

    d_te, t_te = sess.run([daily_returns, total_return], feed_dict={x: test_ins, y_: test_outs})

    t_tr = np.mean(t_tr)
    t_te = np.mean(t_te)

    print("total return train=", "{:.9f}".format(t_tr))
    print("total return test=", "{:.9f}".format(t_te))
    return t_tr, t_te

results = []
pos_results = []
for i in range(100):
    tr, te = run_training_and_return_test_results()
    results.append(te)
    if tr>0:
        pos_results.append(te)
    print(np.mean(results))
    print(np.mean(pos_results))
    #print('final from net', '%f' %zz)

print(np.mean(results))
print(np.mean(pos_results))<commit_msg>modified NNexec<commit_after>#import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import numpy.random as rng
import pandas.io.data as web
import numpy as np
import os
import pandas as pd

# we modify this data organizing slightly to get two symbols
def get_prices(symbol):
    start, end = '2007-05-02', '2016-04-11'
    data = web.DataReader(symbol, 'yahoo', start, end)
    data=pd.DataFrame(data)
    prices=data['Adj Close']
    prices=prices.astype(float)
    return prices

def get_returns(prices):
        return ((prices-prices.shift(-1))/prices)[:-1]

def get_data(list):
    l = []
    for symbol in list:
        rets = get_returns(get_prices(symbol))
        l.append(rets)
    return np.array(l).T

def sort_data(rets):
    ins = []
    outs = []
    for i in range(len(rets)-100):
        ins.append(rets[i:i+100].tolist())
        outs.append(rets[i+100])
    return np.array(ins), np.array(outs)

print('loading data')
symbol_list = ['C', 'GS']
def data_loader_1(symbol_list):
    if os.path.exists('rets.pkl'):
        rets = pd.read_pickle('rets.pkl')
    else:
        rets = get_data(symbol_list)
        pd.to_pickle(rets, 'rets.pkl')
    ins, outs = sort_data(rets)
    ins = ins.transpose([0,2,1]).reshape([-1, len(symbol_list) * 100])
    div = int(.8 * ins.shape[0])
    train_ins, train_outs = ins[:div], outs[:div]
    test_ins, test_outs = ins[div:], outs[div:]

    #normalize inputs
    train_ins, test_ins = train_ins/np.std(ins), test_ins/np.std(ins)
    return train_ins, test_ins, train_outs, test_outs
train_ins, test_ins, train_outs, test_outs = data_loader_1(symbol_list)
print('loaded data')

sess = tf.Session()

positions = tf.constant([-1,0,1]) #long, neutral or short
num_positions = 3
num_symbols = len(symbol_list)
num_samples = 10

n_input = num_symbols * 100
n_hidden_1 = 10 # 1st layer number of features
n_hidden_2 = 10 # 2nd layer number of features
n_classes = num_positions * num_symbols # MNIST total classes (0-9 digits)

# define placeholders
x = tf.placeholder(tf.float32, [None, num_symbols * 100])
y_ = tf.placeholder(tf.float32, [None,  num_symbols])

weights = {
    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
}
biases = {
    'b1': tf.Variable(tf.random_normal([n_hidden_1])),
    'b2': tf.Variable(tf.random_normal([n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_classes]))
}

def multilayer_perceptron(x, weights, biases):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.relu(layer_1)
    # Hidden layer with RELU activation
    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
    layer_2 = tf.nn.relu(layer_2)
    # Output layer with linear activation
    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
    return out_layer

# Construct model
y = multilayer_perceptron(x, weights, biases)

# loop through symbol, taking the columns for each symbol's bucket together
pos = {}
sample_n = {}
sample_mask = {}
symbol_returns = {}
relevant_target_column = {}
for i in range(num_symbols):
    # isolate the buckets relevant to the symbol and get a softmax as well
    symbol_probs = y[:,i*num_positions:(i+1)*num_positions]
    symbol_probs_softmax = tf.nn.softmax(symbol_probs) # softmax[i, j] = exp(logits[i, j]) / sum(exp(logits[i]))
    # sample probability to chose our policy's action
    sample = tf.multinomial(tf.log(symbol_probs_softmax), num_samples)
    for sample_iter in range(num_samples):
        sample_n[i*num_samples + sample_iter] = sample[:,sample_iter]
        pos[i*num_samples + sample_iter] = tf.reshape(sample_n[i*num_samples + sample_iter], [-1]) - 1
        symbol_returns[i*num_samples + sample_iter] = tf.mul(
                                                            tf.cast(pos[i*num_samples + sample_iter], np.float32),
                                                             y_[:,i])

        sample_mask[i*num_samples + sample_iter] = tf.cast(tf.reshape(tf.one_hot(sample_n[i*num_samples + sample_iter], 3), [-1,3]), np.float32)
        relevant_target_column[i*num_samples + sample_iter] = tf.reduce_sum(
                                                    symbol_probs * sample_mask[i*num_samples + sample_iter],1)

daily_returns_by_symbol_ = tf.concat(1, [tf.reshape(t, [-1,1]) for t in symbol_returns.values()])
daily_returns_by_symbol = tf.transpose(tf.reshape(daily_returns_by_symbol_, [-1,2,num_samples]), [0,2,1]) #[?,5,2]
daily_returns = tf.reduce_mean(daily_returns_by_symbol, 2) # [?,5]

total_return = tf.reduce_prod(daily_returns+1, 0)
z = tf.ones_like(total_return) * -1
total_return = tf.add(total_return, z)

ann_vol = tf.mul(
    tf.sqrt(tf.reduce_mean(tf.pow((daily_returns - tf.reduce_mean(daily_returns, 0)),2),0)) ,
    np.sqrt(252)
    )
sharpe = tf.div(total_return, ann_vol)
#Maybe metric slicing later
#segment_ids = tf.ones_like(daily_returns[:,0])
#partial_prod = tf.segment_prod(daily_returns+1, segment_ids)

training_target_cols = tf.concat(1, [tf.reshape(t, [-1,1]) for t in relevant_target_column.values()])
ones = tf.ones_like(training_target_cols)
gradient_ = tf.nn.sigmoid_cross_entropy_with_logits(training_target_cols, ones)

gradient = tf.transpose(tf.reshape(gradient_, [-1,2,num_samples]), [0,2,1]) #[?,5,2]

#cost = tf.mul(gradient , daily_returns_by_symbol_reshaped)
#cost = tf.mul(gradient , tf.expand_dims(daily_returns, -1))
cost = tf.mul(gradient , tf.expand_dims(total_return, -1))
#cost = tf.mul(gradient , tf.expand_dims(sharpe, -1))

optimizer = tf.train.GradientDescentOptimizer(0.0001).minimize(cost)
costfn = tf.reduce_mean(cost)

def run_training_and_return_test_results():
    print('START TRAINING_______________________________')
    # initialize variables to random values
    init = tf.initialize_all_variables()
    sess.run(init)
    # run optimizer on entire training data set many times
    train_size = train_ins.shape[0]
    for epoch in range(20000):
        start = rng.randint(train_size-15)
        batch_size = rng.randint(2,50)
        end = min(train_size, start+batch_size)

        sess.run(optimizer, feed_dict={x: train_ins[start:end], y_: train_outs[start:end]})#.reshape(1,-1).T})
        # every 1000 iterations record progress
        if (epoch+1)%1000== 0:
            t,s, c = sess.run([ total_return, sharpe, costfn], feed_dict={x: train_ins, y_: train_outs})#.reshape(1,-1).T})
            t = np.mean(t)
            s = np.mean(s)
            print("Epoch:", '%04d' % (epoch+1), "cost=",c, "total return=", "{:.9f}".format(t),
                 "sharpe=", "{:.9f}".format(s))
            #print(t)

    print('DONE TRAINING _______________________________')

    d_tr, t_tr = sess.run([daily_returns, total_return], feed_dict={x: train_ins, y_: train_outs})

    d_te, t_te = sess.run([daily_returns, total_return], feed_dict={x: test_ins, y_: test_outs})

    t_tr = np.mean(t_tr)
    t_te = np.mean(t_te)

    print("total return train=", "{:.9f}".format(t_tr))
    print("total return test=", "{:.9f}".format(t_te))
    return t_tr, t_te

results = []
pos_results = []
for i in range(100):
    tr, te = run_training_and_return_test_results()
    results.append(te)
    if tr>0:
        pos_results.append(te)
    print('iter: ', i)
    print(np.mean(results))
    print(np.mean(pos_results))
    #print('final from net', '%f' %zz)

print(np.mean(results))
print(np.mean(pos_results))
<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import
from builtins import str

import os
from os.path import sep, normpath, join, exists
import ntpath
import copy
from collections import namedtuple
import shutil
from subprocess import Popen, PIPE
import re

from tools.arm_pack_manager import Cache
from tools.targets import TARGET_MAP
from tools.export.exporters import Exporter, apply_supported_whitelist
from tools.export.cmsis import DeviceCMSIS

class DeviceUvision(DeviceCMSIS):
    """Uvision Device class, inherits CMSIS Device class

    Encapsulates information necessary for uvision project targets"""
    def __init__(self, target):
        DeviceCMSIS.__init__(self, target)
        dev_format = "$$Device:{0}${1}"
        self.svd = ''
        if self.debug_svd:
            self.svd = dev_format.format(self.dname, self.debug_svd)
        self.reg_file = dev_format.format(self.dname, self.compile_header)
        self.debug_interface = self.uv_debug()
        self.flash_dll = self.generate_flash_dll()

    def uv_debug(self):
        """Return a namedtuple of information about uvision debug settings"""
        UVDebug = namedtuple('UVDebug',['bin_loc','core_flag', 'key'])

        # CortexMXn => pCMX
        cpu = self.core.replace("Cortex-", "C")
        cpu = cpu.replace("+", "")
        cpu = cpu.replace("F", "")
        cpu_flag = "p"+cpu

        # Locations found in Keil_v5/TOOLS.INI
        debuggers = {"st-link": ('STLink\\ST-LINKIII-KEIL_SWO.dll', 'ST-LINKIII-KEIL_SWO'),
                     "j-link":('Segger\\JL2CM3.dll', 'JL2CM3'),
                     "cmsis-dap":('BIN\\CMSIS_AGDI.dll', 'CMSIS_AGDI'),
                     "nulink":('NULink\\Nu_Link.dll','Nu_Link')}
        res = debuggers[self.debug.lower()]
        binary = res[0]
        key = res[1]

        return UVDebug(binary, cpu_flag, key)

    def generate_flash_dll(self):
        '''Flash DLL string from uvision
        S = SW/JTAG Clock ID
        C = CPU index in JTAG chain
        P = Access Port
        For the Options for Target -> Debug tab -> settings -> "Flash" tab in the dialog:
        FD = RAM Start for Flash Functions
        FC = RAM Size for Flash Functions
        FN = Number of Flash types
        FF = Flash File Name (without an extension)
        FS = Start Address of the Flash Device
        FL = Size of the Flash Device
        FP = Full path to the Device algorithm (RTE)

        Necessary to flash some targets. Info gathered from algorithms field of pdsc file.
        '''
        fl_count = 0
        def get_mem_no_x(mem_str):
            mem_reg = "\dx(\w+)"
            m = re.search(mem_reg, mem_str)
            return m.group(1) if m else None

        RAMS = [(get_mem_no_x(info["start"]), get_mem_no_x(info["size"]))
                for mem, info in self.target_info["memory"].items() if "RAM" in mem]
        format_str = "UL2CM3(-S0 -C0 -P0 -FD{ramstart}"+" -FC{ramsize} "+"-FN{num_algos} {extra_flags})"
        ramstart = ''
        #Default according to Keil developer
        ramsize = '1000'
        if len(RAMS)>=1:
            ramstart = RAMS[0][0]
        extra_flags = []
        for name, info in self.target_info["algorithm"].items():
            if not name or not info:
                continue
            if int(info["default"])==0:
                continue
            name_reg = "\w*/([\w_]+)\.flm"
            m = re.search(name_reg, name.lower())
            fl_name = m.group(1) if m else None
            name_flag = "-FF" + str(fl_count) + fl_name

            start, size = get_mem_no_x(info["start"]), get_mem_no_x(info["size"])
            rom_start_flag = "-FS"+str(fl_count)+str(start)
            rom_size_flag = "-FL" + str(fl_count) + str(size)

            if info["ramstart"] is not None and info["ramsize"] is not None:
                ramstart = get_mem_no_x(info["ramstart"])
                ramsize = get_mem_no_x(info["ramsize"])

            path_flag = "-FP" + str(fl_count) + "($$Device:"+self.dname+"$"+name+")"

            extra_flags.extend([name_flag, rom_start_flag, rom_size_flag, path_flag])
            fl_count += 1

        extra = " ".join(extra_flags)
        return format_str.format(ramstart=ramstart,
                                 ramsize=ramsize,
                                 extra_flags=extra, num_algos=fl_count)

class Uvision(Exporter):
    """Keil Uvision class

    This class encapsulates information to be contained in a Uvision
    project file (.uvprojx).
    The needed information can be viewed in uvision.tmpl
    """
    NAME = 'uvision5'
    TOOLCHAIN = 'ARM'

    POST_BINARY_WHITELIST = set([
        "MCU_NRF51Code.binary_hook",
        "TEENSY3_1Code.binary_hook",
        "LPCTargetCode.lpc_patch",
        "LPC4088Code.binary_hook",
        "MTSCode.combine_bins_mts_dot",
        "MTSCode.combine_bins_mts_dragonfly",
        "NCS36510TargetCode.ncs36510_addfib"
    ])

    @classmethod
    def is_target_supported(cls, target_name):
        target = TARGET_MAP[target_name]
        if not (set(target.supported_toolchains) and set(["ARM", "uARM"])):
            return False
        if not DeviceCMSIS.check_supported(target_name):
            return False
        if not hasattr(target, "post_binary_hook"):
            return True
        if target.post_binary_hook['function'] in cls.POST_BINARY_WHITELIST:
            return True
        else:
            return False

    #File associations within .uvprojx file
    file_types = {'.cpp': 8, '.c': 1, '.s': 2,
                  '.obj': 3, '.o': 3, '.lib': 4,
                  '.ar': 4, '.h': 5, '.hpp': 5, '.sct': 4}

    def uv_files(self, files):
        """An generator containing Uvision specific information about project files
        Positional Arguments:
        files - the location of source files

        .uvprojx XML for project file:
        <File>
            <FileType>{{file.type}}</FileType>
            <FileName>{{file.name}}</FileName>
            <FilePath>{{file.loc}}</FilePath>
        </File>
        """
        for loc in files:
            #Encapsulates the information necessary for template entry above
            UVFile = namedtuple('UVFile', ['type','loc','name'])
            _, ext = os.path.splitext(loc)
            if ext.lower() in self.file_types:
                type = self.file_types[ext.lower()]
                name = ntpath.basename(normpath(loc))
                yield UVFile(type, loc, name)

    def format_flags(self):
        """Format toolchain flags for Uvision"""
        flags = copy.deepcopy(self.flags)
        # to be preprocessed with armcc
        asm_flag_string = (
            '--cpreproc --cpreproc_opts=-D__ASSERT_MSG,' +
            ",".join(filter(lambda f: f.startswith("-D"), flags['asm_flags'])))
        flags['asm_flags'] = asm_flag_string
        # All non-asm flags are in one template field
        c_flags = list(set(flags['c_flags'] + flags['cxx_flags'] +flags['common_flags']))
        ld_flags = list(set(flags['ld_flags'] ))
        # These flags are in template to be set by user i n IDE
        template = ["--no_vla", "--cpp", "--c99"]
        # Flag is invalid if set in template
        # Optimizations are also set in the template
        invalid_flag = lambda x: x in template or re.match("-O(\d|time)", x)
        flags['c_flags'] = [flag.replace('"','\\"') for flag in c_flags if not invalid_flag(flag)]
        flags['c_flags'] = " ".join(flags['c_flags'])
        flags['ld_flags'] = " ".join(flags['ld_flags'])
        return flags

    def format_src(self, srcs):
        """Make sources into the named tuple for use in the template"""
        grouped = self.group_project_files(srcs)
        for group, files in grouped.items():
            grouped[group] = sorted(list(self.uv_files(files)),
                                    key=lambda tuple: tuple[2].lower())
        return grouped

    @staticmethod
    def format_fpu(core):
        """Generate a core's FPU string"""
        if core.endswith("FD"):
            return "FPU3(DFPU)"
        elif core.endswith("F"):
            return "FPU2"
        else:
            return ""

    def generate(self):
        """Generate the .uvproj file"""
        cache = Cache(True, False)

        srcs = self.resources.headers + self.resources.s_sources + \
               self.resources.c_sources + self.resources.cpp_sources + \
               self.resources.objects + self.resources.libraries
        ctx = {
            'name': self.project_name,
            # project_files => dict of generators - file group to generator of
            # UVFile tuples defined above
            'project_files': sorted(list(self.format_src(srcs).items()),
                                    key=lambda tuple: tuple[0].lower()),
            'include_paths': ';'.join(self.filter_dot(d) for d in
                                      self.resources.inc_dirs).encode('utf-8'),
            'device': DeviceUvision(self.target),
        }
        sct_file = self.resources.linker_script
        ctx['linker_script'] = self.toolchain.correct_scatter_shebang(
            sct_file, self.resources.file_basepath[sct_file])
        if ctx['linker_script'] != sct_file:
            self.generated_files.append(ctx['linker_script'])
        core = ctx['device'].core
        ctx['cputype'] = core.rstrip("FD")
        if core.endswith("FD"):
            ctx['fpu_setting'] = 3
        elif core.endswith("F"):
            ctx['fpu_setting'] = 2
        else:
            ctx['fpu_setting'] = 1
        ctx['fputype'] = self.format_fpu(core)
        ctx.update(self.format_flags())
        self.gen_file('uvision/uvision.tmpl', ctx, self.project_name+".uvprojx")
        self.gen_file('uvision/uvision_debug.tmpl', ctx, self.project_name + ".uvoptx")

    @staticmethod
    def clean(project_name):
        os.remove(project_name + ".uvprojx")
        os.remove(project_name + ".uvoptx")
        # legacy .build directory cleaned if exists
        if exists('.build'):
            shutil.rmtree('.build')
        if exists('BUILD'):
            shutil.rmtree('BUILD')

    @staticmethod
    def build(project_name, log_name='build_log.txt', cleanup=True):
        """ Build Uvision project """
        # > UV4 -r -j0 -o [log_name] [project_name].uvprojx
        proj_file = project_name + ".uvprojx"
        cmd = ['UV4', '-r', '-j0', '-o', log_name, proj_file]

        # Build the project
        p = Popen(cmd, stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        ret_code = p.returncode

        # Print the log file to stdout
        with open(log_name, 'r') as f:
            print(f.read())

        # Cleanup the exported and built files
        if cleanup:
            os.remove(log_name)
            Uvision.clean(project_name)

        # Returns 0 upon success, 1 upon a warning, and neither upon an error
        if ret_code != 0 and ret_code != 1:
            # Seems like something went wrong.
            return -1
        else:
            return 0
<commit_msg>Compiling Cortex-A cores in uVision is not supported. Reference: http://www2.keil.com/mdk5/selector<commit_after>from __future__ import print_function, absolute_import
from builtins import str

import os
from os.path import sep, normpath, join, exists
import ntpath
import copy
from collections import namedtuple
import shutil
from subprocess import Popen, PIPE
import re

from tools.arm_pack_manager import Cache
from tools.targets import TARGET_MAP
from tools.export.exporters import Exporter, apply_supported_whitelist
from tools.export.cmsis import DeviceCMSIS

class DeviceUvision(DeviceCMSIS):
    """Uvision Device class, inherits CMSIS Device class

    Encapsulates information necessary for uvision project targets"""
    def __init__(self, target):
        DeviceCMSIS.__init__(self, target)
        dev_format = "$$Device:{0}${1}"
        self.svd = ''
        if self.debug_svd:
            self.svd = dev_format.format(self.dname, self.debug_svd)
        self.reg_file = dev_format.format(self.dname, self.compile_header)
        self.debug_interface = self.uv_debug()
        self.flash_dll = self.generate_flash_dll()

    def uv_debug(self):
        """Return a namedtuple of information about uvision debug settings"""
        UVDebug = namedtuple('UVDebug',['bin_loc','core_flag', 'key'])

        # CortexMXn => pCMX
        cpu = self.core.replace("Cortex-", "C")
        cpu = cpu.replace("+", "")
        cpu = cpu.replace("F", "")
        cpu_flag = "p"+cpu

        # Locations found in Keil_v5/TOOLS.INI
        debuggers = {"st-link": ('STLink\\ST-LINKIII-KEIL_SWO.dll', 'ST-LINKIII-KEIL_SWO'),
                     "j-link":('Segger\\JL2CM3.dll', 'JL2CM3'),
                     "cmsis-dap":('BIN\\CMSIS_AGDI.dll', 'CMSIS_AGDI'),
                     "nulink":('NULink\\Nu_Link.dll','Nu_Link')}
        res = debuggers[self.debug.lower()]
        binary = res[0]
        key = res[1]

        return UVDebug(binary, cpu_flag, key)

    def generate_flash_dll(self):
        '''Flash DLL string from uvision
        S = SW/JTAG Clock ID
        C = CPU index in JTAG chain
        P = Access Port
        For the Options for Target -> Debug tab -> settings -> "Flash" tab in the dialog:
        FD = RAM Start for Flash Functions
        FC = RAM Size for Flash Functions
        FN = Number of Flash types
        FF = Flash File Name (without an extension)
        FS = Start Address of the Flash Device
        FL = Size of the Flash Device
        FP = Full path to the Device algorithm (RTE)

        Necessary to flash some targets. Info gathered from algorithms field of pdsc file.
        '''
        fl_count = 0
        def get_mem_no_x(mem_str):
            mem_reg = "\dx(\w+)"
            m = re.search(mem_reg, mem_str)
            return m.group(1) if m else None

        RAMS = [(get_mem_no_x(info["start"]), get_mem_no_x(info["size"]))
                for mem, info in self.target_info["memory"].items() if "RAM" in mem]
        format_str = "UL2CM3(-S0 -C0 -P0 -FD{ramstart}"+" -FC{ramsize} "+"-FN{num_algos} {extra_flags})"
        ramstart = ''
        #Default according to Keil developer
        ramsize = '1000'
        if len(RAMS)>=1:
            ramstart = RAMS[0][0]
        extra_flags = []
        for name, info in self.target_info["algorithm"].items():
            if not name or not info:
                continue
            if int(info["default"])==0:
                continue
            name_reg = "\w*/([\w_]+)\.flm"
            m = re.search(name_reg, name.lower())
            fl_name = m.group(1) if m else None
            name_flag = "-FF" + str(fl_count) + fl_name

            start, size = get_mem_no_x(info["start"]), get_mem_no_x(info["size"])
            rom_start_flag = "-FS"+str(fl_count)+str(start)
            rom_size_flag = "-FL" + str(fl_count) + str(size)

            if info["ramstart"] is not None and info["ramsize"] is not None:
                ramstart = get_mem_no_x(info["ramstart"])
                ramsize = get_mem_no_x(info["ramsize"])

            path_flag = "-FP" + str(fl_count) + "($$Device:"+self.dname+"$"+name+")"

            extra_flags.extend([name_flag, rom_start_flag, rom_size_flag, path_flag])
            fl_count += 1

        extra = " ".join(extra_flags)
        return format_str.format(ramstart=ramstart,
                                 ramsize=ramsize,
                                 extra_flags=extra, num_algos=fl_count)

class Uvision(Exporter):
    """Keil Uvision class

    This class encapsulates information to be contained in a Uvision
    project file (.uvprojx).
    The needed information can be viewed in uvision.tmpl
    """
    NAME = 'uvision5'
    TOOLCHAIN = 'ARM'

    POST_BINARY_WHITELIST = set([
        "MCU_NRF51Code.binary_hook",
        "TEENSY3_1Code.binary_hook",
        "LPCTargetCode.lpc_patch",
        "LPC4088Code.binary_hook",
        "MTSCode.combine_bins_mts_dot",
        "MTSCode.combine_bins_mts_dragonfly",
        "NCS36510TargetCode.ncs36510_addfib"
    ])

    @classmethod
    def is_target_supported(cls, target_name):
        target = TARGET_MAP[target_name]
        if not (set(target.supported_toolchains) and set(["ARM", "uARM"])):
            return False
        if not DeviceCMSIS.check_supported(target_name):
            return False
        if "Cortex-A" in target.core:
            return False
        if not hasattr(target, "post_binary_hook"):
            return True
        if target.post_binary_hook['function'] in cls.POST_BINARY_WHITELIST:
            return True
        else:
            return False

    #File associations within .uvprojx file
    file_types = {'.cpp': 8, '.c': 1, '.s': 2,
                  '.obj': 3, '.o': 3, '.lib': 4,
                  '.ar': 4, '.h': 5, '.hpp': 5, '.sct': 4}

    def uv_files(self, files):
        """An generator containing Uvision specific information about project files
        Positional Arguments:
        files - the location of source files

        .uvprojx XML for project file:
        <File>
            <FileType>{{file.type}}</FileType>
            <FileName>{{file.name}}</FileName>
            <FilePath>{{file.loc}}</FilePath>
        </File>
        """
        for loc in files:
            #Encapsulates the information necessary for template entry above
            UVFile = namedtuple('UVFile', ['type','loc','name'])
            _, ext = os.path.splitext(loc)
            if ext.lower() in self.file_types:
                type = self.file_types[ext.lower()]
                name = ntpath.basename(normpath(loc))
                yield UVFile(type, loc, name)

    def format_flags(self):
        """Format toolchain flags for Uvision"""
        flags = copy.deepcopy(self.flags)
        # to be preprocessed with armcc
        asm_flag_string = (
            '--cpreproc --cpreproc_opts=-D__ASSERT_MSG,' +
            ",".join(filter(lambda f: f.startswith("-D"), flags['asm_flags'])))
        flags['asm_flags'] = asm_flag_string
        # All non-asm flags are in one template field
        c_flags = list(set(flags['c_flags'] + flags['cxx_flags'] +flags['common_flags']))
        ld_flags = list(set(flags['ld_flags'] ))
        # These flags are in template to be set by user i n IDE
        template = ["--no_vla", "--cpp", "--c99"]
        # Flag is invalid if set in template
        # Optimizations are also set in the template
        invalid_flag = lambda x: x in template or re.match("-O(\d|time)", x)
        flags['c_flags'] = [flag.replace('"','\\"') for flag in c_flags if not invalid_flag(flag)]
        flags['c_flags'] = " ".join(flags['c_flags'])
        flags['ld_flags'] = " ".join(flags['ld_flags'])
        return flags

    def format_src(self, srcs):
        """Make sources into the named tuple for use in the template"""
        grouped = self.group_project_files(srcs)
        for group, files in grouped.items():
            grouped[group] = sorted(list(self.uv_files(files)),
                                    key=lambda tuple: tuple[2].lower())
        return grouped

    @staticmethod
    def format_fpu(core):
        """Generate a core's FPU string"""
        if core.endswith("FD"):
            return "FPU3(DFPU)"
        elif core.endswith("F"):
            return "FPU2"
        else:
            return ""

    def generate(self):
        """Generate the .uvproj file"""
        cache = Cache(True, False)

        srcs = self.resources.headers + self.resources.s_sources + \
               self.resources.c_sources + self.resources.cpp_sources + \
               self.resources.objects + self.resources.libraries
        ctx = {
            'name': self.project_name,
            # project_files => dict of generators - file group to generator of
            # UVFile tuples defined above
            'project_files': sorted(list(self.format_src(srcs).items()),
                                    key=lambda tuple: tuple[0].lower()),
            'include_paths': ';'.join(self.filter_dot(d) for d in
                                      self.resources.inc_dirs).encode('utf-8'),
            'device': DeviceUvision(self.target),
        }
        sct_file = self.resources.linker_script
        ctx['linker_script'] = self.toolchain.correct_scatter_shebang(
            sct_file, self.resources.file_basepath[sct_file])
        if ctx['linker_script'] != sct_file:
            self.generated_files.append(ctx['linker_script'])
        core = ctx['device'].core
        ctx['cputype'] = core.rstrip("FD")
        if core.endswith("FD"):
            ctx['fpu_setting'] = 3
        elif core.endswith("F"):
            ctx['fpu_setting'] = 2
        else:
            ctx['fpu_setting'] = 1
        ctx['fputype'] = self.format_fpu(core)
        ctx.update(self.format_flags())
        self.gen_file('uvision/uvision.tmpl', ctx, self.project_name+".uvprojx")
        self.gen_file('uvision/uvision_debug.tmpl', ctx, self.project_name + ".uvoptx")

    @staticmethod
    def clean(project_name):
        os.remove(project_name + ".uvprojx")
        os.remove(project_name + ".uvoptx")
        # legacy .build directory cleaned if exists
        if exists('.build'):
            shutil.rmtree('.build')
        if exists('BUILD'):
            shutil.rmtree('BUILD')

    @staticmethod
    def build(project_name, log_name='build_log.txt', cleanup=True):
        """ Build Uvision project """
        # > UV4 -r -j0 -o [log_name] [project_name].uvprojx
        proj_file = project_name + ".uvprojx"
        cmd = ['UV4', '-r', '-j0', '-o', log_name, proj_file]

        # Build the project
        p = Popen(cmd, stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        ret_code = p.returncode

        # Print the log file to stdout
        with open(log_name, 'r') as f:
            print(f.read())

        # Cleanup the exported and built files
        if cleanup:
            os.remove(log_name)
            Uvision.clean(project_name)

        # Returns 0 upon success, 1 upon a warning, and neither upon an error
        if ret_code != 0 and ret_code != 1:
            # Seems like something went wrong.
            return -1
        else:
            return 0
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import requests
import json
import traceback

from firebase import firebase
from urlparse import urlparse, urljoin
from datetime import datetime
import sqlalchemy
import bleach

from hnapp import app, db
from models.item import Item
from models.lost_item import LostItem
from models.status import Status
from utils import debug_print
from errors import AppError, ScraperError

class Scraper(object):

	firebase = None
	base_url = 'https://hacker-news.firebaseio.com/v0/'

	def connect(self):
		"""
		Connect to firebase API
		You must call this before using the API
		<<< TODO: call this from constructor
		"""
		if not self.firebase:
			self.firebase = firebase.FirebaseApplication(self.base_url, None)

	def save_newest_items(self):
		"""
		Get list of item dicts from the newest items by id
		Includes all acceptable kinds of items (ignores pollopts)
		Saves items in order of ascending ID – important for parent_id and root_id references
		"""

		# Get latest item ids from db and from api
		max_api_id = self.fetch_max_item_id()
		max_db_id = Status.get_max_item_id()

		# If no new data, quit. We don't want to use this function to update items
		if (max_api_id <= max_db_id):
			return

		# Save latest items and update max id
		save = lambda item_data: self.save_item(item_data, update_max_id=True)
		self.fetch_items(range(max_db_id+1, max_api_id), callback=save)

	def save_newest_existing_stories(self, start_from=0, count=100, min_delay=0):
		"""
		Save latest stories
		Does not fetch new stories, uses only those available in database
		Basically this updates story scores
		"""

		# Generate list of newest stories
		# <<< TODO: Actually, we only need ids
		stories = (db.session.query(Item)
							 .with_entities(Item.id)
							 .filter(Item.kind == 'story')
							 .filter(Item.deleted == 0, Item.dead == 0)
							 .order_by(sqlalchemy.desc(Item.id))
							 .slice(start_from, count+start_from)
							 .all()
							 )

		save = lambda item_data: self.save_item(item_data)
		self.fetch_items([item.id for item in stories], callback=save, min_delay=min_delay)

	def save_top_stories(self, front_page, start_from=0, count=100, min_delay=0):
		"""
		Save top stories (ranked by front page order)
		Must indicate whether we're loading front page or not
		"""

		# Generate list of top stories
		story_ids = self.fetch_top_story_ids()[start_from:start_from+count]

		# Save top stories
		save = lambda item_data: self.save_item(item_data, front_page=front_page)
		# def save_item = function(item):
		# 	if (datetime.utcnow() - item.date_updated).total_seconds() > min_delay:
		# 		self.save_item(item_data)
		stories = self.fetch_items(story_ids, callback=save, min_delay=min_delay)

	def save_item(self, item_data, update_max_id=False, front_page=False):
		"""
		Compile and save item to the database
		Each item is committed separately for better fault tolerance
		"""

		# If item was lost
		if isinstance(item_data, LostItem):
			db.session.add(item_data)
			debug_print("Lost %s because %s" % (item_data.id, item_data.reason), '\n')
		else:
			if item_data.get('type', None) in ('story', 'comment', 'poll', 'job', None):
				debug_print("Saving %d" % item_data['id'], '\n')
				compiled_data = self.compile_item_data(item_data, front_page)
				item = Item.create_or_update(compiled_data)
			else:
				debug_print("Skipping %s %d" % (item_data['type'], item_data['id']), '\n')

		if update_max_id:
			Status.set_max_item_id(item_data.id if isinstance(item_data, LostItem) else item_data['id'])

		db.session.commit()

		return item

	def bleach_html(self, html):
		"""
		Sanitize HTML. Leave only safe/expected markup
		"""

		return bleach.clean(
			'
' + html.replace('
', '
\n\n
') + '
',
			tags=('a', 'i', 'p', 'pre'),
			attributes={'a': ['href']},
			styles=(),
			strip=True
			).replace('

', '')

	def compile_item_data(self, raw_item, front_page):
		"""
		Convert raw API output for item to a source dict for an Item model
		"""

		# fields map {api_field: hnapp_field}
		fields = {
			'id': 'id',
			'parent': 'parent_id',
			'type': 'kind',
			'title': 'title',
			'text': 'raw_body',
			'by': 'author',
			'score': 'score',
			'dead': 'dead',
			'deleted': 'deleted'
		}

		item_data = {}

		# Set standard fields listed above
		for raw_field, model_field in fields.iteritems():
			if raw_field in raw_item:
				item_data[model_field] = raw_item[raw_field]

		# Set time
		if 'time' in raw_item:
			item_data['date_posted'] = datetime.fromtimestamp(raw_item['time'])

		# Set sanitized body
		if 'text' in raw_item:
			item_data['body'] = self.bleach_html(raw_item['text'])

		# Set URL and domain
		# Some of the possible "broken" URLs: None, "", " "
		item_data['url'] = None
		item_data['domain'] = None
		if raw_item.get('url', None) is not None:
			parsed_url = urlparse(raw_item['url'])
			if parsed_url.hostname is not None:
				item_data['domain'] = parsed_url.hostname.lower()
				item_data['url'] = raw_item['url'] # <<< should we use urlunparse here? https://docs.python.org/2/library/urlparse.html#urlparse.urlunparse
				if item_data['domain'][:4] == 'www.':
					item_data['domain'] = item_data['domain'][4:]

		# Detect broken stories
		if raw_item.get('type', None) != 'comment' and 'title' not in raw_item:
			item_data['deleted'] = True

		# Set kind and subkind
		# item types map {api_type: [hnapp_kind, hnapp_subkind]}
		item_types = {
			'comment': ['comment', 'comment'],
			'story': ['story', 'link'],
			'poll': ['story', 'poll'],
			'job': ['story', 'job'],
			None: ['story', 'link'] # <<< broken item... maybe these fields should be nullable in DB
		}
		item_data['kind'], item_data['subkind'] = item_types[raw_item.get('type', None)]
		# Special treatment for ask/show stories
		if item_data['subkind'] == 'link' and not item_data.get('deleted', False):
			if 'text' in raw_item and item_data.get('domain', None) is None:
				# Semi-killed items like 8549613 can have no URL, but also don't have a text field
				# Non-broken items on the other hand do have the text field, empty of unapplicable
				item_data['subkind'] = 'ask'
			if item_data['title'].lower()[0:8] == 'show hn:':
				item_data['subkind'] = 'show'

		# Set child ids
		if 'kids' in raw_item:
			item_data['child_ids'] = ','.join(str(child_id) for child_id in raw_item['kids'])

		# Set date when item entered and left front page
		if front_page:
			item_data['date_entered_fp'] = datetime.utcnow()

		# Restore non-deleted and non-dead status
		# Also cast those to int-s for postgresql
		if 'deleted' not in item_data:
			item_data['deleted'] = 0
		else:
			item_data['deleted'] = int(item_data['deleted'])
		if 'dead' not in item_data:
			item_data['dead'] = 0
		else:
			item_data['dead'] = int(item_data['dead'])

		return item_data

	def fetch_max_item_id(self):
		"""
		Fetch max item id available via HN Firebase API
		"""
		debug_print(">> fetch_max_item_id")
		max_id = self.firebase.get('maxitem', None)
		debug_print(max_id, '\n')

		return max_id

	def fetch_item(self, item_id):
		"""
		Fetch item data by id
		Might return an instance of LostItem in case of API or HTTP error
		"""
		debug_print(">> fetch_item %d" % item_id)

		try:
			item = self.firebase.get('item', item_id)
			if item is None:
				item = LostItem(id=item_id,
								reason='null'
								)
			return item
		except requests.exceptions.HTTPError as e:
			# If API error encountered, return a LostItem instead
			lost_item = db.session.query(LostItem).get(item_id)
			if lost_item is None:
				lost_item = LostItem(id=item_id,
									 reason='HTTP/%s' % e.response.status_code,
									 response=e.response.text,
									 traceback=traceback.format_exc()
									 )
			return lost_item

	def fetch_items(self, item_ids, callback=None, min_delay=0):
		"""
		Fetch items for item ids
		Output format: {id1: {item1}, id2: {item2}, ...}

		Failed items might appear as LostItem instances instead of attribute dictionaries
		"""
		debug_print(">> fetch_items")

		items = {}
		for item_id in item_ids:
			# Skip items that have been recently updated, if requested
			if min_delay > 0:
				db_item = db.session.query(Item).get(item_id)
				if db_item is not None and min_delay > (datetime.utcnow() - db_item.date_updated).total_seconds():
					debug_print("Skipped item %d because it's too fresh" % item_id)
					continue
			api_item = self.fetch_item(item_id)
			items[item_id] = api_item
			if callback is not None:
				callback(api_item)

		return items

	def fetch_top_story_ids(self):
		"""
		Fetch stories from top 100 items on front page
		Ordered by current front page rank
		"""
		debug_print(">> fetch_top_story_ids")

		return self.firebase.get('topstories', None)

<commit_msg>Fixed scraping bug for lost and non-included items<commit_after># -*- coding: utf-8 -*-

import requests
import json
import traceback

from firebase import firebase
from urlparse import urlparse, urljoin
from datetime import datetime
import sqlalchemy
import bleach

from hnapp import app, db
from models.item import Item
from models.lost_item import LostItem
from models.status import Status
from utils import debug_print
from errors import AppError, ScraperError

class Scraper(object):

	firebase = None
	base_url = 'https://hacker-news.firebaseio.com/v0/'

	def connect(self):
		"""
		Connect to firebase API
		You must call this before using the API
		<<< TODO: call this from constructor
		"""
		if not self.firebase:
			self.firebase = firebase.FirebaseApplication(self.base_url, None)

	def save_newest_items(self):
		"""
		Get list of item dicts from the newest items by id
		Includes all acceptable kinds of items (ignores pollopts)
		Saves items in order of ascending ID – important for parent_id and root_id references
		"""

		# Get latest item ids from db and from api
		max_api_id = self.fetch_max_item_id()
		max_db_id = Status.get_max_item_id()

		# If no new data, quit. We don't want to use this function to update items
		if (max_api_id <= max_db_id):
			return

		# Save latest items and update max id
		save = lambda item_data: self.save_item(item_data, update_max_id=True)
		self.fetch_items(range(max_db_id+1, max_api_id), callback=save)

	def save_newest_existing_stories(self, start_from=0, count=100, min_delay=0):
		"""
		Save latest stories
		Does not fetch new stories, uses only those available in database
		Basically this updates story scores
		"""

		# Generate list of newest stories
		# <<< TODO: Actually, we only need ids
		stories = (db.session.query(Item)
							 .with_entities(Item.id)
							 .filter(Item.kind == 'story')
							 .filter(Item.deleted == 0, Item.dead == 0)
							 .order_by(sqlalchemy.desc(Item.id))
							 .slice(start_from, count+start_from)
							 .all()
							 )

		save = lambda item_data: self.save_item(item_data)
		self.fetch_items([item.id for item in stories], callback=save, min_delay=min_delay)

	def save_top_stories(self, front_page, start_from=0, count=100, min_delay=0):
		"""
		Save top stories (ranked by front page order)
		Must indicate whether we're loading front page or not
		"""

		# Generate list of top stories
		story_ids = self.fetch_top_story_ids()[start_from:start_from+count]

		# Save top stories
		save = lambda item_data: self.save_item(item_data, front_page=front_page)
		# def save_item = function(item):
		# 	if (datetime.utcnow() - item.date_updated).total_seconds() > min_delay:
		# 		self.save_item(item_data)
		stories = self.fetch_items(story_ids, callback=save, min_delay=min_delay)

	def save_item(self, item_data, update_max_id=False, front_page=False):
		"""
		Compile and save item to the database
		Each item is committed separately for better fault tolerance
		"""

		item = None
		# If item was lost
		if isinstance(item_data, LostItem):
			item = item_data
			db.session.add(item)
			debug_print("Lost %s because %s" % (item_data.id, item_data.reason), '\n')
		else:
			if item_data.get('type', None) in ('story', 'comment', 'poll', 'job', None):
				debug_print("Saving %d" % item_data['id'], '\n')
				compiled_data = self.compile_item_data(item_data, front_page)
				item = Item.create_or_update(compiled_data)
			else:
				debug_print("Skipping %s %d" % (item_data['type'], item_data['id']), '\n')

		if update_max_id:
			Status.set_max_item_id(item_data.id if isinstance(item_data, LostItem) else item_data['id'])

		db.session.commit()

		return item

	def bleach_html(self, html):
		"""
		Sanitize HTML. Leave only safe/expected markup
		"""

		return bleach.clean(
			'
' + html.replace('
', '
\n\n
') + '
',
			tags=('a', 'i', 'p', 'pre'),
			attributes={'a': ['href']},
			styles=(),
			strip=True
			).replace('

', '')

	def compile_item_data(self, raw_item, front_page):
		"""
		Convert raw API output for item to a source dict for an Item model
		"""

		# fields map {api_field: hnapp_field}
		fields = {
			'id': 'id',
			'parent': 'parent_id',
			'type': 'kind',
			'title': 'title',
			'text': 'raw_body',
			'by': 'author',
			'score': 'score',
			'dead': 'dead',
			'deleted': 'deleted'
		}

		item_data = {}

		# Set standard fields listed above
		for raw_field, model_field in fields.iteritems():
			if raw_field in raw_item:
				item_data[model_field] = raw_item[raw_field]

		# Set time
		if 'time' in raw_item:
			item_data['date_posted'] = datetime.fromtimestamp(raw_item['time'])

		# Set sanitized body
		if 'text' in raw_item:
			item_data['body'] = self.bleach_html(raw_item['text'])

		# Set URL and domain
		# Some of the possible "broken" URLs: None, "", " "
		item_data['url'] = None
		item_data['domain'] = None
		if raw_item.get('url', None) is not None:
			parsed_url = urlparse(raw_item['url'])
			if parsed_url.hostname is not None:
				item_data['domain'] = parsed_url.hostname.lower()
				item_data['url'] = raw_item['url'] # <<< should we use urlunparse here? https://docs.python.org/2/library/urlparse.html#urlparse.urlunparse
				if item_data['domain'][:4] == 'www.':
					item_data['domain'] = item_data['domain'][4:]

		# Detect broken stories
		if raw_item.get('type', None) != 'comment' and 'title' not in raw_item:
			item_data['deleted'] = True

		# Set kind and subkind
		# item types map {api_type: [hnapp_kind, hnapp_subkind]}
		item_types = {
			'comment': ['comment', 'comment'],
			'story': ['story', 'link'],
			'poll': ['story', 'poll'],
			'job': ['story', 'job'],
			None: ['story', 'link'] # <<< broken item... maybe these fields should be nullable in DB
		}
		item_data['kind'], item_data['subkind'] = item_types[raw_item.get('type', None)]
		# Special treatment for ask/show stories
		if item_data['subkind'] == 'link' and not item_data.get('deleted', False):
			if 'text' in raw_item and item_data.get('domain', None) is None:
				# Semi-killed items like 8549613 can have no URL, but also don't have a text field
				# Non-broken items on the other hand do have the text field, empty of unapplicable
				item_data['subkind'] = 'ask'
			if item_data['title'].lower()[0:8] == 'show hn:':
				item_data['subkind'] = 'show'

		# Set child ids
		if 'kids' in raw_item:
			item_data['child_ids'] = ','.join(str(child_id) for child_id in raw_item['kids'])

		# Set date when item entered and left front page
		if front_page:
			item_data['date_entered_fp'] = datetime.utcnow()

		# Restore non-deleted and non-dead status
		# Also cast those to int-s for postgresql
		if 'deleted' not in item_data:
			item_data['deleted'] = 0
		else:
			item_data['deleted'] = int(item_data['deleted'])
		if 'dead' not in item_data:
			item_data['dead'] = 0
		else:
			item_data['dead'] = int(item_data['dead'])

		return item_data

	def fetch_max_item_id(self):
		"""
		Fetch max item id available via HN Firebase API
		"""
		debug_print(">> fetch_max_item_id")
		max_id = self.firebase.get('maxitem', None)
		debug_print(max_id, '\n')

		return max_id

	def fetch_item(self, item_id):
		"""
		Fetch item data by id
		Might return an instance of LostItem in case of API or HTTP error
		"""
		debug_print(">> fetch_item %d" % item_id)

		try:
			item = self.firebase.get('item', item_id)
			if item is None:
				item = LostItem(id=item_id,
								reason='null'
								)
			return item
		except requests.exceptions.HTTPError as e:
			# If API error encountered, return a LostItem instead
			lost_item = db.session.query(LostItem).get(item_id)
			if lost_item is None:
				lost_item = LostItem(id=item_id,
									 reason='HTTP/%s' % e.response.status_code,
									 response=e.response.text,
									 traceback=traceback.format_exc()
									 )
			return lost_item

	def fetch_items(self, item_ids, callback=None, min_delay=0):
		"""
		Fetch items for item ids
		Output format: {id1: {item1}, id2: {item2}, ...}

		Failed items might appear as LostItem instances instead of attribute dictionaries
		"""
		debug_print(">> fetch_items")

		items = {}
		for item_id in item_ids:
			# Skip items that have been recently updated, if requested
			if min_delay > 0:
				db_item = db.session.query(Item).get(item_id)
				if db_item is not None and min_delay > (datetime.utcnow() - db_item.date_updated).total_seconds():
					debug_print("Skipped item %d because it's too fresh" % item_id)
					continue
			api_item = self.fetch_item(item_id)
			items[item_id] = api_item
			if callback is not None:
				callback(api_item)

		return items

	def fetch_top_story_ids(self):
		"""
		Fetch stories from top 100 items on front page
		Ordered by current front page rank
		"""
		debug_print(">> fetch_top_story_ids")

		return self.firebase.get('topstories', None)

<|endoftext|>


<commit_before>import exampleclient

from exampleclient.tests import base

class TestExample(base.BaseTestCase):

    def setUp(self):
        super(TestExample, self).setUp()
        print("Setup TestExample")

    def test_hello(self):
        exampleclient.example.hello()

    def test_goodbye(self):
        exampleclient.example.goodbye()
<commit_msg>testing<commit_after>import exampleclient
import exampleclient.example

from exampleclient.tests import base

class TestExample(base.BaseTestCase):

    def setUp(self):
        super(TestExample, self).setUp()
        print("Setup TestExample")

    def test_hello(self):
        exampleclient.example.hello()

    def test_goodbye(self):
        exampleclient.example.goodbye()
<|endoftext|>