<commit_before>#solarnmf_functions.py

#Will Barnes
#27 January 2015

#Description:

#Inputs:

#Outputs:

#Import necessary modules
import numpy as np
import solarnmf_process_data as spd

def make_t_matrix(toption,**kwargs):
    """Set up the observation matrix either using a series of simulated gaussians or a dataset specified by a keyword argument.

    Parameters
    ----------
    toption: str
        Option that says whether this is a simulated ('simulation') observation (a series of gaussians) or an observation to be pulled from a file ('data')
    **kwargs
        nx: int
            number of columns for the simulated data set
        ny: int
            number of rows for the simulated data set
        p: int
            number of sources in the simulated data set

    Returns
    -------
    T: numpy two-dimensional array
        Matrix that represents the observation we want to factorize

    """
    #Check which option to use
    if "simulation" == toption:

        #Parse arguments for simulation
        for key in kwargs:
            if key == "nx":
                nx = int(kwargs[key])
            elif key == "ny":
                ny = int(kwargs[key])
            elif key == "p":
                p = int(kwargs[key])

        #Call function to make simulated results
        sim_result = make_gaussians(nx,ny,p,format=kwargs['format'])

        #If we are using Gauusian matrices, return the matrices
        if kwargs['format'] == 'matrix':
            #Return values
            return {'target':sim_result['target'],'T':sim_result['T']}
        else:
            #Format vector as matrix
            x = sim_result['x']

            #Make sure data is normalized
            x = x/np.max(x)

            #Apply the smoothing function
            x_smooth = spd.smooth_1d_window(x,window_length=21,window='hanning')

            #Make a matrix representation of the time series
            x_mat = spd.ts2mat(x_smooth,int(len(x_smooth)/2),0.05)

            #Rotate the important data along the diagonal
            x_mat_rot = spd.crop_and_rotate(x_mat,45)

            #Return the rotated matrix and the original vector
            return {'T':x_mat_rot,'x':x,'target':sim_result['target']}

    if "data" == toption:
        for key in kwargs:
            if key == "filename":
                filename = kwargs[key]
            elif key == "file_format":
                file_format = kwargs[key]

        #Load the specified file
        x = np.loadtxt(filename)

        #Make sure data is normalized
        x = x/np.max(x)

        #Apply the smoothing function
        x_smooth = spd.smooth_1d_window(x,window_length=21,window='hanning')

        #Make a matrix representation of the time series
        x_mat = spd.ts2mat(x_smooth,len(x_smooth),0.1)

        #Rotate the important data along the diagonal
        x_mat_rot = spd.crop_and_rotate(x_mat,45)

        #Return the rotated matrix and the original vector
        return {'T':x_mat_rot,'x':x}

def make_gaussians(nx,ny,p,**kwargs):

    #Check the format
    if kwargs['format'] == 'matrix':
        #Calculate standard deviations in x and y
        sigma_x = 0.15*np.ones((1,p))
        sigma_y = 0.15*np.ones((1,p))

        #Generate random center positions for pulses (normalized to [0,1])
        centers = np.random.rand(p,2)

        #Preallocate for composite and individual pulses
        T = np.zeros((ny,nx))
        target = np.zeros((ny,nx,p))

        #Set up the X and Y meshes
        X,Y = np.meshgrid(np.linspace(0,1,nx),np.linspace(0,1,ny))

        #Calculate the Gaussian pulses and add them to the T matrix
        for i in range(p):
            #Calculate Gaussian pulse
            z = np.exp(-1/(2*(sigma_x[0,i])**2)*((X - centers[i,0])**2) - 1/(2*(sigma_y[0,i])**2)*((Y - centers[i,1])**2))
            #Add to total T matrix
            T = T + z
            #Save the events that make up the T matrix
            target[:,:,i] = z

        #Normalize the T matrix to one
        T = T/np.max(T)

        #Return the X,Y,T and target matrices
        return {'target':target, 'T':T}
    else:
        #Create list for simulated gaussians
        target = []
        #Initialize total
        x = 0.0
        #Simulate 1D timeseries
        #Create series of Gaussians
        t = np.linspace(-1,1,nx)
        for i in range(p):
            #Get random sign
            if np.random.rand() > 0.5:
                mu_sign = -1.0
            else:
                mu_sign = 1.0
            #Create the Gaussian
            temp = np.random.rand()*np.exp(-(t+mu_sign*np.random.rand())**2/(2.0*0.1**2))
            #Add to total
            x = x + temp
            #Save the components
            target.append(temp)

        #Return the timeseries
        return {'target':target,'x':x}

def initialize_uva(nx,ny,q,r,r_iter,T):

    #Initialize U and V matrices
    utemp = np.random.rand(ny,q)
    vtemp = np.random.rand(q,nx)
    atemp = np.dot(utemp,vtemp)

    #Set artificial value for div
    div_final = 1000
    #Set convergence limit for div_limit
    div_limit = 1.0e-6

    #Begin loop to check divergence criteria
    for i in range(r):
        #Print some output
        print "Starting initialization iteration ",i
        #Call the minimizer
        temp = minimize_div(utemp,vtemp,T,atemp,r_iter,div_limit)
        #Get the last value of the divergence measure
        div_temp = temp['div']
        d_temp = div_temp[-1]

        #Check the new value of div
        if d_temp < div_final:
            div_final = d_temp
            u = temp['u']
            v = temp['v']

        #Generate next random u and v
        utemp = np.random.rand(ny,q)
        vtemp = np.random.rand(q,nx)
        atemp = np.dot(utemp,vtemp)

    #Create the initial a matrix from the initial u and v matrices
    A = np.dot(u,v)

    #Return u and v values with lowest final div value
    return {'u':u, 'v':v, 'A':A}

def minimize_div(u,v,T,A,max_i,div_limit):

    #Initialize vector for divergence metric
    div = np.zeros(max_i)

    #Initialize counter and change in divergence
    i = 0
    delta_div = div_limit + 1

    #Begin loop to minimize divergence metric
    while i < max_i and delta_div > div_limit:

        #Update the U,V, and A matrices
        updates = update_uva(u,v,T,A,i)
        u,v,A = updates['u'],updates['v'],updates['A']

        #Calculate divergence metric
        d = calculate_div(T,A)
        #Save the value
        div[i] = d
        #Calculate the change in the metric
        if i > 0:
            delta_div = np.fabs(d-div[i-1])

        #Print progress (skip for initialization runs)
        if max_i > 20:
            print "i = ",i
            print "div = ",d

        #Increment the counter
        i = i+1

    #Truncate divergence vector
    div = div[0:i]

    #Save the results
    return {'u':u, 'v':v, 'A':A, 'div':div}

def update_uva(u,v,T,A,i):

    #Impose order of update rules (following JDB)
    #Alternating u,v, and A with alternating order
    if i % 2 == 0:
        u = update_u(u,v,T,A)
        A = np.dot(u,v)
        v = update_v(u,v,T,A)
        A = np.dot(u,v)
    else:
        v = update_v(u,v,T,A)
        A = np.dot(u,v)
        u = update_u(u,v,T,A)
        A = np.dot(u,v)

    #Return the matrices
    return {'u':u,'v':v,'A':A}

def update_u(u,v,T,A):
    #Lee & Seung (2000) update rules (for KL divergence)
    m,n = A.shape
    u = u*(np.dot(T/A,np.transpose(v)))/np.dot(np.ones((m,n)),np.transpose(v))

    #Return the u matrix
    return u

def update_v(u,v,T,A):
    #Lee & Seung (2000) update rules (for KL divergence)
    m,n = A.shape
    v = v*(np.dot(np.transpose(u),T/A))/np.dot(np.transpose(u),np.ones((m,n)))

    #Return the v matrix
    return v

def calculate_div(T,A):
    #Calculate the selected divergence metric
    #Kullback-Leibler divergence

    #Initialize the sum
    div = 0

    #Get dimensions of A
    m,n = A.shape

    #Loop to sum
    for i in range(m):
        for j in range(n):

            #Break up into terms
            term_1 = T[i,j]*np.log(T[i,j]/A[i,j])
            if T[i,j] == 0:
                term_1 = 0

            term_2 = -T[i,j]
            term_3 = A[i,j]

            #Add the terms
            div = div + term_1 + term_2 + term_3

    #Return the value
    return div

    <commit_msg>Parameter adjustments<commit_after>#solarnmf_functions.py

#Will Barnes
#27 January 2015

#Description:

#Inputs:

#Outputs:

#Import necessary modules
import numpy as np
import solarnmf_process_data as spd

def make_t_matrix(toption,**kwargs):
    """Set up the observation matrix either using a series of simulated gaussians or a dataset specified by a keyword argument.

    Parameters
    ----------
    toption: str
        Option that says whether this is a simulated ('simulation') observation (a series of gaussians) or an observation to be pulled from a file ('data')
    **kwargs
        nx: int
            number of columns for the simulated data set
        ny: int
            number of rows for the simulated data set
        p: int
            number of sources in the simulated data set

    Returns
    -------
    T: numpy two-dimensional array
        Matrix that represents the observation we want to factorize

    """
    #Check which option to use
    if "simulation" == toption:

        #Parse arguments for simulation
        for key in kwargs:
            if key == "nx":
                nx = int(kwargs[key])
            elif key == "ny":
                ny = int(kwargs[key])
            elif key == "p":
                p = int(kwargs[key])

        #Call function to make simulated results
        sim_result = make_gaussians(nx,ny,p,format=kwargs['format'])

        #If we are using Gauusian matrices, return the matrices
        if kwargs['format'] == 'matrix':
            #Return values
            return {'target':sim_result['target'],'T':sim_result['T']}
        else:
            #Format vector as matrix
            x = sim_result['x']

            #Make sure data is normalized
            x = x/np.max(x)

            #Apply the smoothing function
            x_smooth = spd.smooth_1d_window(x,window_length=21,window='hanning')

            #Make a matrix representation of the time series
            x_mat = spd.ts2mat(x_smooth,len(x_smooth),0.1)

            #Rotate the important data along the diagonal
            x_mat_rot = spd.crop_and_rotate(x_mat,45)

            #Return the rotated matrix and the original vector
            return {'T':x_mat_rot,'x':x,'target':sim_result['target']}

    if "data" == toption:
        for key in kwargs:
            if key == "filename":
                filename = kwargs[key]
            elif key == "file_format":
                file_format = kwargs[key]

        #Load the specified file
        x = np.loadtxt(filename)

        #Make sure data is normalized
        x = x/np.max(x)

        #Apply the smoothing function
        x_smooth = spd.smooth_1d_window(x,window_length=21,window='hanning')

        #Make a matrix representation of the time series
        x_mat = spd.ts2mat(x_smooth,len(x_smooth),0.1)

        #Rotate the important data along the diagonal
        x_mat_rot = spd.crop_and_rotate(x_mat,45)

        #Return the rotated matrix and the original vector
        return {'T':x_mat_rot,'x':x}

def make_gaussians(nx,ny,p,**kwargs):

    #Check the format
    if kwargs['format'] == 'matrix':
        #Calculate standard deviations in x and y
        sigma_x = 0.15*np.ones((1,p))
        sigma_y = 0.15*np.ones((1,p))

        #Generate random center positions for pulses (normalized to [0,1])
        centers = np.random.rand(p,2)

        #Preallocate for composite and individual pulses
        T = np.zeros((ny,nx))
        target = np.zeros((ny,nx,p))

        #Set up the X and Y meshes
        X,Y = np.meshgrid(np.linspace(0,1,nx),np.linspace(0,1,ny))

        #Calculate the Gaussian pulses and add them to the T matrix
        for i in range(p):
            #Calculate Gaussian pulse
            z = np.exp(-1/(2*(sigma_x[0,i])**2)*((X - centers[i,0])**2) - 1/(2*(sigma_y[0,i])**2)*((Y - centers[i,1])**2))
            #Add to total T matrix
            T = T + z
            #Save the events that make up the T matrix
            target[:,:,i] = z

        #Normalize the T matrix to one
        T = T/np.max(T)

        #Return the X,Y,T and target matrices
        return {'target':target, 'T':T}
    else:
        #Create list for simulated gaussians
        target = []
        #Initialize total
        x = 0.0
        #Simulate 1D timeseries
        #Create series of Gaussians
        t = np.linspace(-1,1,nx)
        for i in range(p):
            #Get random sign
            if np.random.rand() > 0.5:
                mu_sign = -1.0
            else:
                mu_sign = 1.0
            #Create the Gaussian
            temp = np.random.rand()*np.exp(-(t+mu_sign*np.random.rand())**2/(2.0*0.1**2))
            #Add to total
            x = x + temp
            #Save the components
            target.append(temp)

        #Return the timeseries
        return {'target':target,'x':x}

def initialize_uva(nx,ny,q,r,r_iter,T):

    #Initialize U and V matrices
    utemp = np.random.rand(ny,q)
    vtemp = np.random.rand(q,nx)
    atemp = np.dot(utemp,vtemp)

    #Set artificial value for div
    div_final = 1000
    #Set convergence limit for div_limit
    div_limit = 1.0e-6

    #Begin loop to check divergence criteria
    for i in range(r):
        #Print some output
        print "Starting initialization iteration ",i
        #Call the minimizer
        temp = minimize_div(utemp,vtemp,T,atemp,r_iter,div_limit)
        #Get the last value of the divergence measure
        div_temp = temp['div']
        d_temp = div_temp[-1]

        #Check the new value of div
        if d_temp < div_final:
            div_final = d_temp
            u = temp['u']
            v = temp['v']

        #Generate next random u and v
        utemp = np.random.rand(ny,q)
        vtemp = np.random.rand(q,nx)
        atemp = np.dot(utemp,vtemp)

    #Create the initial a matrix from the initial u and v matrices
    A = np.dot(u,v)

    #Return u and v values with lowest final div value
    return {'u':u, 'v':v, 'A':A}

def minimize_div(u,v,T,A,max_i,div_limit):

    #Initialize vector for divergence metric
    div = np.zeros(max_i)

    #Initialize counter and change in divergence
    i = 0
    delta_div = div_limit + 1

    #Begin loop to minimize divergence metric
    while i < max_i and delta_div > div_limit:

        #Update the U,V, and A matrices
        updates = update_uva(u,v,T,A,i)
        u,v,A = updates['u'],updates['v'],updates['A']

        #Calculate divergence metric
        d = calculate_div(T,A)
        #Save the value
        div[i] = d
        #Calculate the change in the metric
        if i > 0:
            delta_div = np.fabs(d-div[i-1])

        #Print progress (skip for initialization runs)
        if max_i > 20:
            print "i = ",i
            print "div = ",d

        #Increment the counter
        i = i+1

    #Truncate divergence vector
    div = div[0:i]

    #Save the results
    return {'u':u, 'v':v, 'A':A, 'div':div}

def update_uva(u,v,T,A,i):

    #Impose order of update rules (following JDB)
    #Alternating u,v, and A with alternating order
    if i % 2 == 0:
        u = update_u(u,v,T,A)
        A = np.dot(u,v)
        v = update_v(u,v,T,A)
        A = np.dot(u,v)
    else:
        v = update_v(u,v,T,A)
        A = np.dot(u,v)
        u = update_u(u,v,T,A)
        A = np.dot(u,v)

    #Return the matrices
    return {'u':u,'v':v,'A':A}

def update_u(u,v,T,A):
    #Lee & Seung (2000) update rules (for KL divergence)
    m,n = A.shape
    u = u*(np.dot(T/A,np.transpose(v)))/np.dot(np.ones((m,n)),np.transpose(v))

    #Return the u matrix
    return u

def update_v(u,v,T,A):
    #Lee & Seung (2000) update rules (for KL divergence)
    m,n = A.shape
    v = v*(np.dot(np.transpose(u),T/A))/np.dot(np.transpose(u),np.ones((m,n)))

    #Return the v matrix
    return v

def calculate_div(T,A):
    #Calculate the selected divergence metric
    #Kullback-Leibler divergence

    #Initialize the sum
    div = 0

    #Get dimensions of A
    m,n = A.shape

    #Loop to sum
    for i in range(m):
        for j in range(n):

            #Break up into terms
            term_1 = T[i,j]*np.log(T[i,j]/A[i,j])
            if T[i,j] == 0:
                term_1 = 0

            term_2 = -T[i,j]
            term_3 = A[i,j]

            #Add the terms
            div = div + term_1 + term_2 + term_3

    #Return the value
    return div

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Assembly routines
-----------------

Functions to assemble the system of equations for a finite element
analysis.

"""
from __future__ import absolute_import, division, print_function
import numpy as np
from scipy.sparse import coo_matrix
import solidspy.uelutil as ue
import solidspy.femutil as fem

def eqcounter(cons, ndof_node=2):
    """Count active equations

    Creates boundary conditions array bc_array

    Parameters
    ----------
    cons : ndarray.
      Array with constraints for each node.

    Returns
    -------
    neq : int
      Number of equations in the system after removing the nodes
      with imposed displacements.
    bc_array : ndarray (int)
      Array that maps the nodes with number of equations.

    """
    nnodes = cons.shape[0]
    bc_array = cons.copy().astype(int)
    neq = 0
    for i in range(nnodes):
        for j in range(ndof_node):
            if bc_array[i, j] == 0:
                bc_array[i, j] = neq
                neq += 1

    return neq, bc_array

def DME(cons, elements, ndof_node=2):
    """Create assembly array operator

    Count active equations, create boundary conditions array ``bc_array``
    and the assembly operator ``assem_op``.

    Parameters
    ----------
    cons : ndarray.
      Array with constraints for each degree of freedom in each node.
    elements : ndarray
      Array with the number for the nodes in each element.

    Returns
    -------
    assem_op : ndarray (int)
      Assembly operator.
    bc_array : ndarray (int)
      Boundary conditions array.
    neq : int
      Number of active equations in the system.

    """
    nels = elements.shape[0]
    assem_op = np.zeros([nels, 18], dtype=np.integer)
    neq, bc_array = eqcounter(cons, ndof_node=2)
    for ele in range(nels):
        iet = elements[ele, 1]
        ndof, _, _ = fem.eletype(iet)
        assem_op[ele, :ndof] = bc_array[elements[ele, 3:]].flatten()
    return assem_op, bc_array, neq

def ele_fun(eletype):
    """Return the function for the element type given

    Parameters
    ----------
    eletype : int
      Type of element.

    Returns
    -------
    ele_fun : callable
      Function for the element type given.
    """
    elem_id = {
        1: ue.uel4nquad,
        2: ue.uel6ntrian,
        3: ue.uel3ntrian,
        5: ue.uelspring,
        6: ue.ueltruss2D,
        7: ue.uelbeam2DU}
    try:
        return elem_id[eletype]
    except:
        raise ValueError("You entered an invalid type of element.")

def retriever(elements, mats, nodes, ele, uel=None):
    """Computes the elemental stiffness matrix of element ``ele``

    Parameters
    ----------
    elements : ndarray
      Array with the number for the nodes in each element.
    mats : ndarray.
      Array with the material profiles.
    nodes : ndarray.
      Array with the nodal numbers and coordinates.
    ele : int.
      Identifier of the element to be assembled.

    Returns
    -------
    kloc : ndarray (float)
      Array with the local stiffness matrix.
    mloc : ndarray (float)
      Array with the local mass matrix.
    """
    elem_type = elements[ele, 1]
    params = mats[elements[ele, 2], :]
    elcoor = nodes[elements[ele, 3:], 1:3]
    if uel is None:
        uel = ele_fun(elem_type)
    kloc, mloc = uel(elcoor, params)
    return kloc, mloc

def assembler(elements, mats, nodes, neq, assem_op, sparse=True, uel=None):
    """Assembles the global stiffness matrix

    Parameters
    ----------
    elements : ndarray (int)
      Array with the number for the nodes in each element.
    mats    : ndarray (float)
      Array with the material profiles.
    nodes    : ndarray (float)
      Array with the nodal numbers and coordinates.
    assem_op : ndarray (int)
      Assembly operator.
    neq : int
      Number of active equations in the system.
    sparse : boolean (optional)
      Boolean variable to pick sparse assembler. It is True
      by default.
    uel : callable function (optional)
      Python function that returns the local stiffness matrix.

    Returns
    -------
    kglob : ndarray (float)
      Array with the global stiffness matrix. It might be
      dense or sparse, depending on the value of _sparse_
    mglob : ndarray (float)
      Array with the global mass matrix. It might be
      dense or sparse, depending on the value of _sparse_

    """
    if sparse:
        kglob, mglob = sparse_assem(elements, mats, nodes, neq, assem_op,
                                    uel=uel)
    else:
        kglob, mglob = dense_assem(elements, mats, nodes, neq, assem_op,
                                   uel=uel)

    return kglob, mglob

def dense_assem(elements, mats, nodes, neq, assem_op, uel=None):
    """
    Assembles the global stiffness matrix
    using a dense storing scheme

    Parameters
    ----------
    elements : ndarray (int)
      Array with the number for the nodes in each element.
    mats    : ndarray (float)
      Array with the material profiles.
    nodes    : ndarray (float)
      Array with the nodal numbers and coordinates.
    assem_op : ndarray (int)
      Assembly operator.
    neq : int
      Number of active equations in the system.
    uel : callable function (optional)
      Python function that returns the local stiffness matrix.

    Returns
    -------
    kglob : ndarray (float)
      Array with the global stiffness matrix in a dense numpy
      array.

    """
    kglob = np.zeros((neq, neq))
    mglob = np.zeros((neq, neq))
    nels = elements.shape[0]
    for ele in range(nels):
        kloc, mloc = retriever(elements, mats, nodes, ele, uel=uel)
        ndof = kloc.shape[0]
        dme = assem_op[ele, :ndof]
        for row in range(ndof):
            glob_row = dme[row]
            if glob_row != -1:
                for col in range(ndof):
                    glob_col = dme[col]
                    if glob_col != -1:
                        kglob[glob_row, glob_col] += kloc[row, col]
                        mglob[glob_row, glob_col] += mloc[row, col]

    return kglob, mglob

def sparse_assem(elements, mats, nodes, neq, assem_op, uel=None):
    """
    Assembles the global stiffness matrix
    using a sparse storing scheme

    The scheme used to assemble is COOrdinate list (COO), and
    it converted to Compressed Sparse Row (CSR) afterward
    for the solution phase [1]_.

    Parameters
    ----------
    elements : ndarray (int)
      Array with the number for the nodes in each element.
    mats    : ndarray (float)
      Array with the material profiles.
    nodes    : ndarray (float)
      Array with the nodal numbers and coordinates.
    assem_op : ndarray (int)
      Assembly operator.
    neq : int
      Number of active equations in the system.
    uel : callable function (optional)
      Python function that returns the local stiffness matrix.

    Returns
    -------
    kglob : sparse matrix (float)
      Array with the global stiffness matrix in a sparse
      Compressed Sparse Row (CSR) format.

    References
    ----------
    .. [1] Sparse matrix. (2017, March 8). In Wikipedia,
        The Free Encyclopedia.
        https://en.wikipedia.org/wiki/Sparse_matrix

    """
    rows = []
    cols = []
    stiff_vals = []
    mass_vals = []
    nels = elements.shape[0]
    for ele in range(nels):
        kloc, mloc = retriever(elements, mats, nodes, ele, uel=uel)
        ndof = kloc.shape[0]
        dme = assem_op[ele, :ndof]
        for row in range(ndof):
            glob_row = dme[row]
            if glob_row != -1:
                for col in range(ndof):
                    glob_col = dme[col]
                    if glob_col != -1:
                        rows.append(glob_row)
                        cols.append(glob_col)
                        stiff_vals.append(kloc[row, col])
                        mass_vals.append(mloc[row, col])

    stiff = coo_matrix((stiff_vals, (rows, cols)), shape=(neq, neq)).tocsr()
    mass = coo_matrix((mass_vals, (rows, cols)), shape=(neq, neq)).tocsr()
    return stiff, mass

def loadasem(loads, bc_array, neq):
    """Assembles the global Right Hand Side Vector

    Parameters
    ----------
    loads : ndarray
      Array with the loads imposed in the system.
    bc_array : ndarray (int)
      Array that maps the nodes with number of equations.
    neq : int
      Number of equations in the system after removing the nodes
      with imposed displacements.

    Returns
    -------
    rhs_vec : ndarray
      Array with the right hand side vector.

    """
    nloads = loads.shape[0]
    rhs_vec = np.zeros([neq])
    for cont in range(nloads):
        il = int(loads[cont, 0])
        ilx = bc_array[il, 0]
        ily = bc_array[il, 1]
        if ilx != -1:
            rhs_vec[ilx] = loads[cont, 1]
        if ily != -1:
            rhs_vec[ily] = loads[cont, 2]

    return rhs_vec

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>Allow to assemble variable degrees of freedom per node<commit_after># -*- coding: utf-8 -*-
"""
Assembly routines
-----------------

Functions to assemble the system of equations for a finite element
analysis.

"""
from __future__ import absolute_import, division, print_function
import numpy as np
from scipy.sparse import coo_matrix
import solidspy.uelutil as ue
import solidspy.femutil as fem

def eqcounter(cons, ndof_node=2):
    """Count active equations

    Creates boundary conditions array bc_array

    Parameters
    ----------
    cons : ndarray.
      Array with constraints for each node.

    Returns
    -------
    neq : int
      Number of equations in the system after removing the nodes
      with imposed displacements.
    bc_array : ndarray (int)
      Array that maps the nodes with number of equations.

    """
    nnodes = cons.shape[0]
    bc_array = cons.copy().astype(int)
    neq = 0
    for i in range(nnodes):
        for j in range(ndof_node):
            if bc_array[i, j] == 0:
                bc_array[i, j] = neq
                neq += 1

    return neq, bc_array

def DME(cons, elements, ndof_node=2):
    """Create assembly array operator

    Count active equations, create boundary conditions array ``bc_array``
    and the assembly operator ``assem_op``.

    Parameters
    ----------
    cons : ndarray.
      Array with constraints for each degree of freedom in each node.
    elements : ndarray
      Array with the number for the nodes in each element.

    Returns
    -------
    assem_op : ndarray (int)
      Assembly operator.
    bc_array : ndarray (int)
      Boundary conditions array.
    neq : int
      Number of active equations in the system.

    """
    nels = elements.shape[0]
    assem_op = np.zeros([nels, 18], dtype=np.integer)
    neq, bc_array = eqcounter(cons, ndof_node=ndof_node)
    for ele in range(nels):
        iet = elements[ele, 1]
        ndof, _, _ = fem.eletype(iet)
        assem_op[ele, :ndof] = bc_array[elements[ele, 3:]].flatten()
    return assem_op, bc_array, neq

def ele_fun(eletype):
    """Return the function for the element type given

    Parameters
    ----------
    eletype : int
      Type of element.

    Returns
    -------
    ele_fun : callable
      Function for the element type given.
    """
    elem_id = {
        1: ue.uel4nquad,
        2: ue.uel6ntrian,
        3: ue.uel3ntrian,
        5: ue.uelspring,
        6: ue.ueltruss2D,
        7: ue.uelbeam2DU}
    try:
        return elem_id[eletype]
    except:
        raise ValueError("You entered an invalid type of element.")

def retriever(elements, mats, nodes, ele, uel=None):
    """Computes the elemental stiffness matrix of element ``ele``

    Parameters
    ----------
    elements : ndarray
      Array with the number for the nodes in each element.
    mats : ndarray.
      Array with the material profiles.
    nodes : ndarray.
      Array with the nodal numbers and coordinates.
    ele : int.
      Identifier of the element to be assembled.

    Returns
    -------
    kloc : ndarray (float)
      Array with the local stiffness matrix.
    mloc : ndarray (float)
      Array with the local mass matrix.
    """
    elem_type = elements[ele, 1]
    params = mats[elements[ele, 2], :]
    elcoor = nodes[elements[ele, 3:], 1:3]
    if uel is None:
        uel = ele_fun(elem_type)
    kloc, mloc = uel(elcoor, params)
    return kloc, mloc

def assembler(elements, mats, nodes, neq, assem_op, sparse=True, uel=None):
    """Assembles the global stiffness matrix

    Parameters
    ----------
    elements : ndarray (int)
      Array with the number for the nodes in each element.
    mats    : ndarray (float)
      Array with the material profiles.
    nodes    : ndarray (float)
      Array with the nodal numbers and coordinates.
    assem_op : ndarray (int)
      Assembly operator.
    neq : int
      Number of active equations in the system.
    sparse : boolean (optional)
      Boolean variable to pick sparse assembler. It is True
      by default.
    uel : callable function (optional)
      Python function that returns the local stiffness matrix.

    Returns
    -------
    kglob : ndarray (float)
      Array with the global stiffness matrix. It might be
      dense or sparse, depending on the value of _sparse_
    mglob : ndarray (float)
      Array with the global mass matrix. It might be
      dense or sparse, depending on the value of _sparse_

    """
    if sparse:
        kglob, mglob = sparse_assem(elements, mats, nodes, neq, assem_op,
                                    uel=uel)
    else:
        kglob, mglob = dense_assem(elements, mats, nodes, neq, assem_op,
                                   uel=uel)

    return kglob, mglob

def dense_assem(elements, mats, nodes, neq, assem_op, uel=None):
    """
    Assembles the global stiffness matrix
    using a dense storing scheme

    Parameters
    ----------
    elements : ndarray (int)
      Array with the number for the nodes in each element.
    mats    : ndarray (float)
      Array with the material profiles.
    nodes    : ndarray (float)
      Array with the nodal numbers and coordinates.
    assem_op : ndarray (int)
      Assembly operator.
    neq : int
      Number of active equations in the system.
    uel : callable function (optional)
      Python function that returns the local stiffness matrix.

    Returns
    -------
    kglob : ndarray (float)
      Array with the global stiffness matrix in a dense numpy
      array.

    """
    kglob = np.zeros((neq, neq))
    mglob = np.zeros((neq, neq))
    nels = elements.shape[0]
    for ele in range(nels):
        kloc, mloc = retriever(elements, mats, nodes, ele, uel=uel)
        ndof = kloc.shape[0]
        dme = assem_op[ele, :ndof]
        for row in range(ndof):
            glob_row = dme[row]
            if glob_row != -1:
                for col in range(ndof):
                    glob_col = dme[col]
                    if glob_col != -1:
                        kglob[glob_row, glob_col] += kloc[row, col]
                        mglob[glob_row, glob_col] += mloc[row, col]

    return kglob, mglob

def sparse_assem(elements, mats, nodes, neq, assem_op, uel=None):
    """
    Assembles the global stiffness matrix
    using a sparse storing scheme

    The scheme used to assemble is COOrdinate list (COO), and
    it converted to Compressed Sparse Row (CSR) afterward
    for the solution phase [1]_.

    Parameters
    ----------
    elements : ndarray (int)
      Array with the number for the nodes in each element.
    mats    : ndarray (float)
      Array with the material profiles.
    nodes    : ndarray (float)
      Array with the nodal numbers and coordinates.
    assem_op : ndarray (int)
      Assembly operator.
    neq : int
      Number of active equations in the system.
    uel : callable function (optional)
      Python function that returns the local stiffness matrix.

    Returns
    -------
    kglob : sparse matrix (float)
      Array with the global stiffness matrix in a sparse
      Compressed Sparse Row (CSR) format.

    References
    ----------
    .. [1] Sparse matrix. (2017, March 8). In Wikipedia,
        The Free Encyclopedia.
        https://en.wikipedia.org/wiki/Sparse_matrix

    """
    rows = []
    cols = []
    stiff_vals = []
    mass_vals = []
    nels = elements.shape[0]
    for ele in range(nels):
        kloc, mloc = retriever(elements, mats, nodes, ele, uel=uel)
        ndof = kloc.shape[0]
        dme = assem_op[ele, :ndof]
        for row in range(ndof):
            glob_row = dme[row]
            if glob_row != -1:
                for col in range(ndof):
                    glob_col = dme[col]
                    if glob_col != -1:
                        rows.append(glob_row)
                        cols.append(glob_col)
                        stiff_vals.append(kloc[row, col])
                        mass_vals.append(mloc[row, col])

    stiff = coo_matrix((stiff_vals, (rows, cols)), shape=(neq, neq)).tocsr()
    mass = coo_matrix((mass_vals, (rows, cols)), shape=(neq, neq)).tocsr()
    return stiff, mass

def loadasem(loads, bc_array, neq):
    """Assembles the global Right Hand Side Vector

    Parameters
    ----------
    loads : ndarray
      Array with the loads imposed in the system.
    bc_array : ndarray (int)
      Array that maps the nodes with number of equations.
    neq : int
      Number of equations in the system after removing the nodes
      with imposed displacements.

    Returns
    -------
    rhs_vec : ndarray
      Array with the right hand side vector.

    """
    nloads = loads.shape[0]
    rhs_vec = np.zeros([neq])
    for cont in range(nloads):
        il = int(loads[cont, 0])
        ilx = bc_array[il, 0]
        ily = bc_array[il, 1]
        if ilx != -1:
            rhs_vec[ilx] = loads[cont, 1]
        if ily != -1:
            rhs_vec[ily] = loads[cont, 2]

    return rhs_vec

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>import logging
import os
import sqlite3
import numpy as np
import time
from os.path import dirname, abspath
from abc import ABC, abstractmethod

from src import Utils
# Utils.set_keras_backend("tensorflow")

project_root_dir = dirname(dirname(dirname(abspath(__file__))))
log_dir = os.path.join(project_root_dir, 'logs')
db_path = os.path.join(project_root_dir, 'data', 'db.sqlite')
if not os.path.exists(db_path):
    print('no db found at: ' + db_path)
    exit(1)

class BaseLearn(ABC):

    def __init__(self):
        self.db = sqlite3.connect(db_path)
        cursor = self.db.cursor()
        self.logger = Utils.get_unique_file_logger(self, logging.INFO)
        self.numb_all_games = cursor.execute(
            'SELECT COUNT(*) FROM meta').fetchone()[0]
        self.games_table_length = cursor.execute(
            'SELECT COUNT(*) FROM games').fetchone()[0]
        self.invalid_game_ids = cursor.execute(
            'SELECT id FROM meta WHERE all_moves_imported=0').fetchall()
        self.log('''database contains {} games,
            {} are invalid and won\'t be used for training'''.format(
            self.numb_all_games, len(self.invalid_game_ids)))
        self.training_size = self.games_table_length  # override this in your Learn class as desired

    def log(self, msg):
        self.logger.info(msg)
        print(msg)

    # can be used by Learn classes extending this class that require more game-info during training
    def get_sgf(self, game_id):
        return self.db.cursor().execute('SELECT sgf_content FROM meta WHERE id=?', (game_id,)).fetchone()[0]

    @staticmethod
    def append_to_numpy_array(base, new):
        if base is None:
            base = np.array([new])
        else:
            base = np.vstack((base, new))
        return base

    @abstractmethod
    def handle_data(self, training_data):
        pass

    @abstractmethod
    def setup_and_compile_model(self):
        pass

    @abstractmethod
    def train(self, model, X, Y):
        pass

    @abstractmethod
    def get_path_to_self(self):
        pass

    def run(self):
        start_time = time.time()
        # self.log('starting the training with moves from '
        #          + ('all ' if self.numb_games_to_learn_from == self.numb_all_games else '')
        #          + str(self.numb_games_to_learn_from) + ' games as input ' + self.get_path_to_self())

        # Get data from Database
        cursor = self.db.cursor()
        cursor.execute('''SELECT games.*
                          FROM games, meta
                          WHERE games.id == meta.id
                          AND meta.all_moves_imported!=0
                          LIMIT ?''',
                       [self.training_size])
        training_data = np.array(cursor.fetchall())  # this is a gigantic array, has millions of rows

        self.log('working with {} rows'.format(len(training_data)))
        X, Y = self.handle_data(training_data)

        # SET UP and STORE NETWORK TOPOLOGY as json
        model = self.setup_and_compile_model()
        architecture_path = os.path.join(dirname(self.get_path_to_self()), 'model_architecture.json')
        json_file = open(architecture_path, 'w')
        json_file.write(model.to_json())
        json_file.close()

        # TRAIN and then STORE WEIGHTS as hdf5
        self.train(model, X, Y)
        weights_path = os.path.join(dirname(self.get_path_to_self()), 'model_weights.h5')
        model.save_weights(weights_path)

        # EVALUATE
        scores = model.evaluate(X, Y)
        print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))

        # DONE
        elapsed_time = time.time() - start_time
        self.log('training ended after {:.0f}s'.format(elapsed_time))
        self.log('model trained on {} moves from {} games'.format(
                 len(X), self.numb_all_games))
        self.log('model architecture saved to: ' + architecture_path)
        self.log('model weights saved to: ' + weights_path)
<commit_msg>info-print was wrong<commit_after>import logging
import os
import sqlite3
import numpy as np
import time
from os.path import dirname, abspath
from abc import ABC, abstractmethod

from src import Utils
# Utils.set_keras_backend("tensorflow")

project_root_dir = dirname(dirname(dirname(abspath(__file__))))
log_dir = os.path.join(project_root_dir, 'logs')
db_path = os.path.join(project_root_dir, 'data', 'db.sqlite')
if not os.path.exists(db_path):
    print('no db found at: ' + db_path)
    exit(1)

class BaseLearn(ABC):

    def __init__(self):
        self.db = sqlite3.connect(db_path)
        cursor = self.db.cursor()
        self.logger = Utils.get_unique_file_logger(self, logging.INFO)
        self.numb_all_games = cursor.execute(
            'SELECT COUNT(*) FROM meta').fetchone()[0]
        self.games_table_length = cursor.execute(
            'SELECT COUNT(*) FROM games').fetchone()[0]
        self.invalid_game_ids = cursor.execute(
            'SELECT id FROM meta WHERE all_moves_imported=0').fetchall()
        self.log('''database contains {} games,
            {} are invalid and won\'t be used for training'''.format(
            self.numb_all_games, len(self.invalid_game_ids)))
        self.training_size = self.games_table_length  # override this in your Learn class as desired

    def log(self, msg):
        self.logger.info(msg)
        print(msg)

    # can be used by Learn classes extending this class that require more game-info during training
    def get_sgf(self, game_id):
        return self.db.cursor().execute('SELECT sgf_content FROM meta WHERE id=?', (game_id,)).fetchone()[0]

    @staticmethod
    def append_to_numpy_array(base, new):
        if base is None:
            base = np.array([new])
        else:
            base = np.vstack((base, new))
        return base

    @abstractmethod
    def handle_data(self, training_data):
        pass

    @abstractmethod
    def setup_and_compile_model(self):
        pass

    @abstractmethod
    def train(self, model, X, Y):
        pass

    @abstractmethod
    def get_path_to_self(self):
        pass

    def run(self):
        start_time = time.time()
        # self.log('starting the training with moves from '
        #          + ('all ' if self.numb_games_to_learn_from == self.numb_all_games else '')
        #          + str(self.numb_games_to_learn_from) + ' games as input ' + self.get_path_to_self())

        # Get data from Database
        cursor = self.db.cursor()
        cursor.execute('''SELECT games.*
                          FROM games, meta
                          WHERE games.id == meta.id
                          AND meta.all_moves_imported!=0
                          LIMIT ?''',
                       [self.training_size])
        training_data = np.array(cursor.fetchall())  # this is a gigantic array, has millions of rows

        self.log('working with {} rows'.format(len(training_data)))
        X, Y = self.handle_data(training_data)

        # SET UP and STORE NETWORK TOPOLOGY as json
        model = self.setup_and_compile_model()
        architecture_path = os.path.join(dirname(self.get_path_to_self()), 'model_architecture.json')
        json_file = open(architecture_path, 'w')
        json_file.write(model.to_json())
        json_file.close()

        # TRAIN and then STORE WEIGHTS as hdf5
        self.train(model, X, Y)
        weights_path = os.path.join(dirname(self.get_path_to_self()), 'model_weights.h5')
        model.save_weights(weights_path)

        # EVALUATE
        scores = model.evaluate(X, Y)
        print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))

        # DONE
        elapsed_time = time.time() - start_time
        self.log('training ended after {:.0f}s'.format(elapsed_time))
        self.log('model trained on {} moves, 8 symmetries included that\'s a training size of {}'
                 .format(self.training_size, len(X)))
        self.log('model architecture saved to: ' + architecture_path)
        self.log('model weights saved to: ' + weights_path)
<|endoftext|>


<commit_before>"""
WSGI config for Inthe.AM's status server.
"""

import datetime
import json
import logging
import pickle
from queue import Queue
import time
import urllib.parse as urlparse
from wsgiref import util as wsgiref_utils

import django

django.setup()  # noqa

from django.conf import settings
from django.core.signing import Signer

from gevent import sleep

from inthe_am.taskmanager.lock import get_lock_redis

logger = logging.getLogger("inthe_am.wsgi_status")
logging.config.dictConfig(settings.LOGGING)

def get_announcements_subscription(store, username, channels):
    client = get_lock_redis()
    subscription = client.pubsub(ignore_subscribe_messages=True)

    final_channels = []

    for channel in channels:
        final_channels.append(channel.format(username=username.encode("utf8")))

    subscription.subscribe(*channels)

    return subscription

def sse_offload(env, start_response):
    app = Application(env, start_response)

    for row in app.generator():
        yield row.encode("utf-8")

class Application:
    HEADERS = [
        ("Content-Type", "text/event-stream"),
    ]
    ERROR_RETRY_DELAY = 60 * 1000

    def add_message(self, name, data=None):
        if data is None:
            data = ""

        self.queue.put(
            {"name": name, "data": data,}
        )

    def handle_local_sync(self, message):
        new_head = json.loads(message["data"])["head"]

        self.add_message("local_sync", message["data"])

        if new_head != self.head:
            self.head = new_head
            self.add_message("head_changed", self.head)

    def handle_changed_task(self, message):
        self.add_message("task_changed", json.loads(message["data"])["task_id"])

    def handle_log_message(self, message):
        announcement = json.loads(message["data"])
        if announcement["error"] and not announcement["silent"]:
            self.add_message("error_logged", announcement["message"])

    def handle_personal_announcement(self, message):
        self.add_message("personal_announcement", message["data"])

    def handle_public_announcement(self, message):
        message_data = json.loads(message["data"])

        # Open the envelope, see if it's a system announcement or not.
        if not message_data.get("system", False):
            self.add_message("public_announcement", message["data"])
        else:
            self.add_message(message_data["type"], json.dumps(message_data["data"]))

    def handle_message(self, message):
        channel = message["channel"].decode("utf-8")
        if channel.startswith("local_sync."):
            self.handle_local_sync(message)
            return True
        elif channel.startswith("changed_task."):
            self.handle_changed_task(message)
            return True
        elif channel.startswith("log_message."):
            self.handle_log_message(message)
            return True
        elif channel.startswith("personal."):
            self.handle_personal_announcement(message)
            return True
        elif channel == settings.ANNOUNCEMENTS_CHANNEL:
            return True

        return False

    def beat_heart(self):
        heartbeat_interval = datetime.timedelta(
            seconds=settings.EVENT_STREAM_HEARTBEAT_INTERVAL
        )
        if (
            not self.last_heartbeat
            or self.last_heartbeat + heartbeat_interval < datetime.datetime.now()
        ):
            self.add_message("heartbeat")
            self.last_heartbeat = datetime.datetime.now()

    def __init__(self, env, start_response):
        start_response("200 OK", self.HEADERS)

        self.last_heartbeat = None
        self.env = env
        self.response = env
        self.signer = Signer()
        self.initialized = False
        self.queue = Queue()

        client = get_lock_redis()
        pickled_data = pickle.loads(client.get(f"pickle_{env['PICKLE_ID']}"))
        self.store = pickled_data["taskstore"]
        self.username = pickled_data["username"]

        try:
            logger.info(
                "Starting event stream for TaskStore %s for user %s",
                self.store.pk,
                self.username,
            )
            query = urlparse.parse_qs(
                urlparse.urlparse(wsgiref_utils.request_uri(env)).query
            )
            try:
                self.head = query["head"][0]
            except (KeyError, IndexError):
                self.head = self.store.repository.head().decode("utf-8")

            # Subscribe to the event stream
            self.subscription = get_announcements_subscription(
                self.store,
                self.username,
                [
                    "local_sync.{username}",
                    "changed_task.{username}",
                    "log_message.{username}",
                    "personal.{username}",
                    settings.ANNOUNCEMENTS_CHANNEL,
                ],
            )

            # Kick-off a sync just to be sure
            kwargs = {"asynchronous": True, "function": ("views.Status.iterator")}
            self.store.sync(msg="Iterator initialization", **kwargs)

            # Let the client know the head has changed if they've asked
            # for a different head than the one we're on:
            if self.head != self.store.repository.head().decode("utf-8"):
                for task_id in self.store.get_changed_task_ids(self.head):
                    self.add_message("task_changed", task_id)

            self.initialized = True

        except Exception as e:
            logger.exception("Error starting event stream: %s", str(e))

    def generator(self):
        try:
            if not self.initialized:
                yield f"retry: {self.ERROR_RETRY_DELAY}\n\n"
                return

            self.beat_heart()
            created = time.time()
            while time.time() - created < settings.EVENT_STREAM_TIMEOUT:
                self.beat_heart()

                # Queue-up all messages that have occurred
                while True:
                    message = self.subscription.get_message()
                    if not message:
                        break

                    self.handle_message(message)

                # Emit queued messages
                while not self.queue.empty():
                    message = self.queue.get(False)
                    if not message:
                        continue

                    if message.get("name"):
                        yield f"event: {message['name']}\n"
                    yield f"data: {message.get('data', '')}\n"
                    yield "\n"

                # Relax
                sleep(settings.EVENT_STREAM_LOOP_INTERVAL)
            self.subscription.unsubscribe()
            self.subscription.close()
        except Exception:
            self.subscription.unsubscribe()
            self.subscription.close()
            raise

try:
    application = sse_offload
except Exception as e:
    logging.exception("Event stream terminated by uncaught exception: %s", str(e))
<commit_msg>Fixing SSE channel subscriptions; this has been broken for a while without anybody noticing...<commit_after>"""
WSGI config for Inthe.AM's status server.
"""

import datetime
import json
import logging
import logging.config
import pickle
from queue import Queue
import time
import urllib.parse as urlparse
from wsgiref import util as wsgiref_utils

import django

django.setup()  # noqa

from django.conf import settings  # noqa

from gevent import sleep  # noqa

from inthe_am.taskmanager.lock import get_lock_redis  # noqa

logger = logging.getLogger("inthe_am.wsgi_status")
logging.config.dictConfig(settings.LOGGING)

def get_announcements_subscription(store, username, channels):
    client = get_lock_redis()
    subscription = client.pubsub(ignore_subscribe_messages=True)

    final_channels = []

    for channel in channels:
        final_channels.append(channel)

    subscription.subscribe(*channels)

    return subscription

def sse_offload(env, start_response):
    app = Application(env, start_response)

    for row in app.generator():
        yield row.encode("utf-8")

class Application:
    HEADERS = [
        ("Content-Type", "text/event-stream"),
    ]
    ERROR_RETRY_DELAY = 60 * 1000

    def add_message(self, name, data=None):
        if data is None:
            data = ""

        logger.debug("Adding message %s: %s", name, data)

        self.queue.put(
            {"name": name, "data": data,}
        )

    def handle_local_sync(self, message):
        new_head = json.loads(message["data"])["head"]

        self.add_message("local_sync", message["data"])

        if new_head != self.head:
            self.head = new_head
            self.add_message("head_changed", self.head)

    def handle_changed_task(self, message):
        self.add_message("task_changed", json.loads(message["data"])["task_id"])

    def handle_log_message(self, message):
        announcement = json.loads(message["data"])
        if announcement["error"] and not announcement["silent"]:
            self.add_message("error_logged", announcement["message"])

    def handle_personal_announcement(self, message):
        self.add_message("personal_announcement", message["data"])

    def handle_public_announcement(self, message):
        message_data = json.loads(message["data"])

        # Open the envelope, see if it's a system announcement or not.
        if not message_data.get("system", False):
            self.add_message("public_announcement", message["data"])
        else:
            self.add_message(message_data["type"], json.dumps(message_data["data"]))

    def handle_message(self, message):
        channel = message["channel"].decode("utf-8")
        if channel.startswith("local_sync."):
            self.handle_local_sync(message)
            return True
        elif channel.startswith("changed_task."):
            self.handle_changed_task(message)
            return True
        elif channel.startswith("log_message."):
            self.handle_log_message(message)
            return True
        elif channel.startswith("personal."):
            self.handle_personal_announcement(message)
            return True
        elif channel == settings.ANNOUNCEMENTS_CHANNEL:
            return True

        return False

    def beat_heart(self):
        heartbeat_interval = datetime.timedelta(
            seconds=settings.EVENT_STREAM_HEARTBEAT_INTERVAL
        )
        if (
            not self.last_heartbeat
            or self.last_heartbeat + heartbeat_interval < datetime.datetime.now()
        ):
            self.add_message("heartbeat")
            self.last_heartbeat = datetime.datetime.now()

    def __init__(self, env, start_response):
        start_response("200 OK", self.HEADERS)

        self.last_heartbeat = None
        self.initialized = False
        self.queue = Queue()

        client = get_lock_redis()
        pickled_data = pickle.loads(client.get(f"pickle_{env['PICKLE_ID']}"))
        self.store = pickled_data["taskstore"]
        self.username = pickled_data["username"]

        try:
            logger.info(
                "Starting event stream for TaskStore %s for user %s",
                self.store.pk,
                self.username,
            )
            query = urlparse.parse_qs(
                urlparse.urlparse(wsgiref_utils.request_uri(env)).query
            )
            try:
                self.head = query["head"][0]
            except (KeyError, IndexError):
                self.head = self.store.repository.head().decode("utf-8")

            # Subscribe to the event stream
            self.subscription = get_announcements_subscription(
                self.store,
                self.username,
                [
                    f"local_sync.{self.username}",
                    f"changed_task.{self.username}",
                    f"log_message.{self.username}",
                    f"personal.{self.username}",
                    settings.ANNOUNCEMENTS_CHANNEL,
                ],
            )

            # Kick-off a sync just to be sure
            kwargs = {"asynchronous": True, "function": ("views.Status.iterator")}
            self.store.sync(msg="Iterator initialization", **kwargs)

            # Let the client know the head has changed if they've asked
            # for a different head than the one we're on:
            if self.head != self.store.repository.head().decode("utf-8"):
                for task_id in self.store.get_changed_task_ids(self.head):
                    self.add_message("task_changed", task_id)

            self.initialized = True

        except Exception as e:
            logger.exception("Error starting event stream: %s", str(e))

    def generator(self):
        try:
            if not self.initialized:
                yield f"retry: {self.ERROR_RETRY_DELAY}\n\n"
                return

            self.beat_heart()
            created = time.time()
            while time.time() - created < settings.EVENT_STREAM_TIMEOUT:
                self.beat_heart()

                # Queue-up all messages that have occurred
                while True:
                    message = self.subscription.get_message()
                    logger.debug("Found message: %s", message)
                    if not message:
                        break

                    self.handle_message(message)

                # Emit queued messages
                while not self.queue.empty():
                    message = self.queue.get(False)
                    if not message:
                        continue

                    logger.debug("Emitting %s on bus", message)
                    if message.get("name"):
                        yield f"event: {message['name']}\n"
                    yield f"data: {message.get('data', '')}\n"
                    yield "\n"

                # Relax
                sleep(settings.EVENT_STREAM_LOOP_INTERVAL)
            self.subscription.unsubscribe()
            self.subscription.close()
        except Exception:
            self.subscription.unsubscribe()
            self.subscription.close()
            raise

try:
    application = sse_offload
except Exception as e:
    logging.exception("Event stream terminated by uncaught exception: %s", str(e))
<|endoftext|>


<commit_before>__version_info__ = (0, 12, 0)
__version__ = '.'.join(map(str, __version_info__))
<commit_msg>Bump version for dev/next release<commit_after>__version_info__ = (0, 13, 0)
__version__ = '.'.join(map(str, __version_info__))
<|endoftext|>


<commit_before># discoursegraphs.readwrite: input/output functionality

"""
The ``readwrite`` package contains importers, exporters and other
output functionality. Basically, it allows you to convert annotated
linguistic documents into a graph-based representation for further
processing.
"""

__all__ = ['TigerDocumentGraph', 'AnaphoraDocumentGraph', 'RSTGraph']

from discoursegraphs.readwrite.tiger import TigerDocumentGraph
from discoursegraphs.readwrite.anaphoricity import AnaphoraDocumentGraph
from discoursegraphs.readwrite.rst import RSTGraph
<commit_msg>added MMAXDocumentGraph to readwrite.__init__<commit_after># discoursegraphs.readwrite: input/output functionality

"""
The ``readwrite`` package contains importers, exporters and other
output functionality. Basically, it allows you to convert annotated
linguistic documents into a graph-based representation for further
processing.
"""

__all__ = [
    'TigerDocumentGraph', 'AnaphoraDocumentGraph', 'RSTGraph',
    'MMAXDocumentGraph'
]

from discoursegraphs.readwrite.tiger import TigerDocumentGraph
from discoursegraphs.readwrite.anaphoricity import AnaphoraDocumentGraph
from discoursegraphs.readwrite.rst import RSTGraph
from discoursegraphs.readwrite.mmax2 import MMAXDocumentGraph
<|endoftext|>


<commit_before>__version__ = '0.0.1'
<commit_msg>Version bump<commit_after>__version__ = '0.0.2'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from json import loads
from collections.abc import Mapping

from elasticsearch import Elasticsearch

from intelmq.lib.bot import Bot

def replace_keys(obj, key_char='.', replacement='_'):
    if isinstance(obj, Mapping):
        replacement_obj = {}
        for key, val in obj.items():
            replacement_key = key.replace(key_char, replacement)
            replacement_obj[replacement_key] = replace_keys(val, key_char, replacement)
        return replacement_obj
    return obj

class ElasticsearchOutputBot(Bot):

    def init(self):
        self.elastic_host = getattr(self.parameters,
                                    'elastic_host', '127.0.0.1')
        self.elastic_port = getattr(self.parameters,
                                    'elastic_port', '9200')
        self.elastic_index = getattr(self.parameters,
                                     'elastic_index', 'intelmq')
        self.elastic_doctype = getattr(self.parameters,
                                       'elastic_doctype', 'events')
        self.replacement_char = getattr(self.parameters,
                                        'replacement_char', '_')
        self.flatten_fields = getattr(self.parameters,
                                      'flatten_fields', [])
        if isinstance(self.flatten_fields, str):
            self.flatten_fields = self.flatten_fields.split(',')

        self.es = Elasticsearch([{'host': self.elastic_host,
                                  'port': self.elastic_port}])
        if not self.es.indices.exists(self.elastic_index):
            self.es.indices.create(index=self.elastic_index, ignore=400)

    def process(self):
        event = self.receive_message()
        event_dict = event.to_dict(hierarchical=False)

        for field in self.flatten_fields:
            if field in event_dict:
                val = event_dict[field]
                # if it string try to convert to json
                if isinstance(val, str):
                    try:
                        val = loads(val)
                    except:
                        pass
                if isinstance(val, Mapping):
                    for key, value in val.items():
                        event_dict[key] = value
                    event_dict.pop(field)

        event_dict = replace_keys(event_dict,
                                  replacement=self.replacement_char)

        self.es.index(index=self.elastic_index,
                      doc_type=self.elastic_doctype,
                      body=event_dict)
        self.acknowledge_message()

BOT = ElasticsearchOutputBot
<commit_msg>pep8 fix<commit_after># -*- coding: utf-8 -*-

from json import loads
from collections.abc import Mapping

from elasticsearch import Elasticsearch

from intelmq.lib.bot import Bot

def replace_keys(obj, key_char='.', replacement='_'):
    if isinstance(obj, Mapping):
        replacement_obj = {}
        for key, val in obj.items():
            replacement_key = key.replace(key_char, replacement)
            replacement_obj[replacement_key] = replace_keys(val, key_char, replacement)
        return replacement_obj
    return obj

class ElasticsearchOutputBot(Bot):

    def init(self):
        self.elastic_host = getattr(self.parameters,
                                    'elastic_host', '127.0.0.1')
        self.elastic_port = getattr(self.parameters,
                                    'elastic_port', '9200')
        self.elastic_index = getattr(self.parameters,
                                     'elastic_index', 'intelmq')
        self.elastic_doctype = getattr(self.parameters,
                                       'elastic_doctype', 'events')
        self.replacement_char = getattr(self.parameters,
                                        'replacement_char', '_')
        self.flatten_fields = getattr(self.parameters,
                                      'flatten_fields', [])
        if isinstance(self.flatten_fields, str):
            self.flatten_fields = self.flatten_fields.split(',')

        self.es = Elasticsearch([{'host': self.elastic_host,
                                  'port': self.elastic_port}])
        if not self.es.indices.exists(self.elastic_index):
            self.es.indices.create(index=self.elastic_index, ignore=400)

    def process(self):
        event = self.receive_message()
        event_dict = event.to_dict(hierarchical=False)

        for field in self.flatten_fields:
            if field in event_dict:
                val = event_dict[field]
                # if it string try to convert to json
                if isinstance(val, str):
                    try:
                        val = loads(val)
                    except:
                        pass
                if isinstance(val, Mapping):
                    for key, value in val.items():
                        event_dict[key] = value
                    event_dict.pop(field)

        event_dict = replace_keys(event_dict,
                                  replacement=self.replacement_char)

        self.es.index(index=self.elastic_index,
                      doc_type=self.elastic_doctype,
                      body=event_dict)
        self.acknowledge_message()

BOT = ElasticsearchOutputBot
<|endoftext|>


<commit_before>from core.colorMessage import dyeWARNING, dyeFAIL, dyeOKGREEN, dyeOKBLUE
from core.misc import human_size
from prettytable import PrettyTable
import os
import datetime
import functools
import pdb

def build_list_if(boolean, func):
    if boolean:
        return func()
    else:
        return []

def format_project_tbl(projects, size=False, cost=False):
    def get_size():
        row_size = p.size_stat()
        row_size = [row_size[k] for k in ('clean', 'project')]
        return row_size

    build_size_field = functools.partial(build_list_if, boolean = size, func = lambda: ['clean Data', 'project data'])
    build_size = functools.partial(build_list_if, boolean = size, func = get_size)
    build_cost_field = functools.partial(build_list_if, boolean = cost, func = lambda: ['cost'])
    build_cost = functools.partial(build_list_if, boolean = cost, func = lambda: [p.cost()])

    def build_row(p):
        state = states[p.name]
        progress = 100.0 * (state.get('cleaned', 0) + state.get('finished', 0)) / sum(state.values())
        progress = round(progress, 2)
        elapsed = diff_date(get_date(p.start_date), get_date(p.finish_date))
        row_size = build_size()
        row_cost = build_cost()
        if size and row_cost:
            row_cost = [sum(row_size) * 0.148 + row_cost[0]]
        return [p.name] + [state.get(column, 0) for column in states_column] + [progress, elapsed] + row_size + row_cost

    tbl = PrettyTable()
    states = {e.name:e.states() for e in projects}
    states_column = set(sum([state.keys() for state in states.values()], []))
    size_field = build_size_field()
    cost_field = build_cost_field()
    tbl.field_names = ['project'] + list(states_column) + ['progress(%)', 'elapsed'] + size_field + cost_field

    for p in projects:
        row = build_row(p)
        tbl.add_row(row)
    return tbl

def format_detail_porject(project):
    tbl = PrettyTable()
    tbl.header = False
    fields = ['id', 'name', 'description', 'owner', 'status', 'type', 'pipe', 'path', 'max_job', 'run_cnt', 'create_date', 'start_date', 'finish_date', 'discount', 'email', 'mns', 'cluster']
    values = [project.__getattribute__(k) for k in fields]
    elapsed = diff_date(get_date(project.start_date), get_date(project.finish_date))
    tbl.add_column("field", fields + ['task num', 'elapsed'])
    tbl.add_column("value", values + [len(project.task), elapsed])
    return tbl

def format_tasks_tbl(tasks, cost=False):
    build_cost_field = functools.partial(build_list_if, boolean = cost, func = lambda: ['cost'])
    build_cost = functools.partial(build_list_if, boolean = cost, func = lambda: [task.cost()])

    tbl = PrettyTable()
    cost_field = build_cost_field()
    tbl.field_names = ['id', 'name', 'status', 'failed', 'module', 'app', 'instance', 'create', 'waited', 'elapsed'] + cost_field
    for task in tasks:
        if task.bcs:
            failed_cnts = len([b for b in task.bcs if b.status == 'Failed'])
            create_date = get_date(task.bcs[-1].create_date)
            start_date = get_date(task.bcs[-1].start_date)
            finish_date = get_date(task.bcs[-1].finish_date)
            waited = diff_date(create_date, start_date)
            elapsed = diff_date(start_date, finish_date)
        else:
            failed_cnts = 0
            create_date = task.project.create_date.replace(microsecond=0)
            waited = elapsed = None
        cost = build_cost()
        status = format_status(task.aasm_state, task.aasm_state == 'cleaned')
        row = [task.id, os.path.basename(task.shell), status, failed_cnts, task.module.name, task.app.name, task.instance.name, create_date, waited, elapsed] + cost
        tbl.add_row(row)
    return tbl

def get_date(date):
    if date:
        return date.replace(microsecond=0)
    elif date is None:
        return None
    else:
        raise ValueError("Invalid datetime")

def diff_date(t1, t2):
    if all([t1, t2]):
        return t2 - t1
    elif t1 and not t2:
        now = datetime.datetime.now().replace(microsecond=0)
        return now - t1
    else:
        return None

def format_detail_task(task):
    tbl = PrettyTable()
    tbl.header = False
    fields = ['id', 'name', 'status', 'module', 'app', 'instance', 'docker_image', 'cpu', 'mem', 'disk_type', 'disk_size', 'debug_mode', 'benchmark']
    status = format_status(task.aasm_state, task.aasm_state == 'cleaned')
    values = [task.id, os.path.basename(task.shell), status, task.module.name, task.app.name, task.instance.name, task.docker_image, task.cpu, task.mem, task.disk_type, task.disk_size, task.debug_mode, task.benchmark]
    tbl.add_column("field", fields)
    tbl.add_column("value", values)
    return tbl

def format_bcs_tbl(bcs, with_instance):
    tbl = PrettyTable()
    fields = ['id', 'name', 'status', 'create', 'waited', 'elapsed', 'cost']
    if with_instance:
        fields = fields + ['instance', 'cpu', 'mem', 'instance price', 'spot price']
    tbl.field_names = fields
    for b in bcs:
        status = format_status(b.status.lower(), b.deleted)
        create_date = get_date(b.create_date)
        start_date = get_date(b.start_date)
        finish_date = get_date(b.finish_date)
        waited = diff_date(create_date, start_date)
        elapsed = diff_date(start_date, finish_date)
        row = [b.id, b.name, status, create_date, waited, elapsed, b.cost]
        if with_instance:
            i = b.instance
            row = row + [i.name, i.cpu, i.mem, i.price, b.spot_price_limit]
        tbl.add_row(row)
    return tbl

def format_status(status, deleted):
    if status == 'failed':
        status = dyeFAIL(status)
    elif status == 'finished':
        status = dyeOKGREEN(status)
    elif status == 'stopped':
        status = dyeWARNING(status)
    else:
        status = dyeOKBLUE(status)

    if deleted:
        status = status + " (D)"
    return status

def format_mapping_tbl(mappings, size=False):
    tbl = PrettyTable()
    fields = ['id', 'name', 'source', 'destination', 'is_write', 'is_immediate', 'is_required']
    if size:
        fields += ['size']
    tbl.field_names = fields
    tbl.align['source'] = "l"
    tbl.align['destination'] = "l"
    tbl.align['size'] = "r"
    for m in mappings:
        if m.exists():
            destination = dyeOKGREEN(m.destination)
        else:
            destination = dyeFAIL(m.destination)
        row = [m.id, m.name, m.source, destination, m.is_write, m.is_immediate, m.is_required]
        if size:
            row += [human_size(m.size())]
        tbl.add_row(row)

    return tbl

def format_detail_mapping(mapping, size=False):
    tbl = PrettyTable()
    tbl.header = False
    fields = ['id', 'name', 'source', 'destination', 'is_write', 'is_immediate', 'is_required']
    values = [mapping.__getattribute__(k) for k in fields]

    if mapping.exists():
        values[3] = dyeOKGREEN(values[3])
    else:
        values[3] = dyeFAIL(values[3])

    if size:
        fields += ['size']
        values += [human_size(mapping.size())]

    tbl.add_column("field", fields)
    tbl.add_column("value", values)
    return tbl

def format_instance_tbl(instances, latest_price = False):
    tbl = PrettyTable()
    fields = ['name', 'cpu', 'mem', 'disk type', 'disk size', 'price']
    if latest_price:
        fields += ['spot price', 'origin price', 'discount']
    tbl.field_names = fields
    for i in instances:
        row = [i.name, i.cpu, i.mem, i.disk_type, i.disk_size, i.price]
        if latest_price:
            (spot_price, origin_price) = i.latest_price()
            if origin_price:
                discount = spot_price / origin_price
            else:
                discount = None
            row += [spot_price, origin_price, discount]
        tbl.add_row(row)

    return tbl

def format_cost_tbl(costs):
    tbl = PrettyTable()
    tbl.field_names = ['id', 'name', 'size', 'size cost', 'bcs cost', 'total']
    map(tbl.add_row, costs)
    return tbl

def format_history_price_tbl(prices):
    tbl = PrettyTable()
    tbl.field_names = ['InstanceType', 'Timestamp', 'ZoneId', 'IoOptimized', 'NetworkType', 'OriginPrice', 'SpotPrice', 'Discount']
    for p in prices:
        p['Discount'] = round(p['SpotPrice'] / p['OriginPrice'], 2)
        tbl.add_row([p[key] for key in tbl.field_names])

    return tbl
<commit_msg>add source color for format_mapping_tbl<commit_after>from core.colorMessage import dyeWARNING, dyeFAIL, dyeOKGREEN, dyeOKBLUE
from core.misc import human_size
from prettytable import PrettyTable
import os
import datetime
import functools
import pdb

def build_list_if(boolean, func):
    if boolean:
        return func()
    else:
        return []

def format_project_tbl(projects, size=False, cost=False):
    def get_size():
        row_size = p.size_stat()
        row_size = [row_size[k] for k in ('clean', 'project')]
        return row_size

    build_size_field = functools.partial(build_list_if, boolean = size, func = lambda: ['clean Data', 'project data'])
    build_size = functools.partial(build_list_if, boolean = size, func = get_size)
    build_cost_field = functools.partial(build_list_if, boolean = cost, func = lambda: ['cost'])
    build_cost = functools.partial(build_list_if, boolean = cost, func = lambda: [p.cost()])

    def build_row(p):
        state = states[p.name]
        progress = 100.0 * (state.get('cleaned', 0) + state.get('finished', 0)) / sum(state.values())
        progress = round(progress, 2)
        elapsed = diff_date(get_date(p.start_date), get_date(p.finish_date))
        row_size = build_size()
        row_cost = build_cost()
        if size and row_cost:
            row_cost = [sum(row_size) * 0.148 + row_cost[0]]
        return [p.name] + [state.get(column, 0) for column in states_column] + [progress, elapsed] + row_size + row_cost

    tbl = PrettyTable()
    states = {e.name:e.states() for e in projects}
    states_column = set(sum([state.keys() for state in states.values()], []))
    size_field = build_size_field()
    cost_field = build_cost_field()
    tbl.field_names = ['project'] + list(states_column) + ['progress(%)', 'elapsed'] + size_field + cost_field

    for p in projects:
        row = build_row(p)
        tbl.add_row(row)
    return tbl

def format_detail_porject(project):
    tbl = PrettyTable()
    tbl.header = False
    fields = ['id', 'name', 'description', 'owner', 'status', 'type', 'pipe', 'path', 'max_job', 'run_cnt', 'create_date', 'start_date', 'finish_date', 'discount', 'email', 'mns', 'cluster']
    values = [project.__getattribute__(k) for k in fields]
    elapsed = diff_date(get_date(project.start_date), get_date(project.finish_date))
    tbl.add_column("field", fields + ['task num', 'elapsed'])
    tbl.add_column("value", values + [len(project.task), elapsed])
    return tbl

def format_tasks_tbl(tasks, cost=False):
    build_cost_field = functools.partial(build_list_if, boolean = cost, func = lambda: ['cost'])
    build_cost = functools.partial(build_list_if, boolean = cost, func = lambda: [task.cost()])

    tbl = PrettyTable()
    cost_field = build_cost_field()
    tbl.field_names = ['id', 'name', 'status', 'failed', 'module', 'app', 'instance', 'create', 'waited', 'elapsed'] + cost_field
    for task in tasks:
        if task.bcs:
            failed_cnts = len([b for b in task.bcs if b.status == 'Failed'])
            create_date = get_date(task.bcs[-1].create_date)
            start_date = get_date(task.bcs[-1].start_date)
            finish_date = get_date(task.bcs[-1].finish_date)
            waited = diff_date(create_date, start_date)
            elapsed = diff_date(start_date, finish_date)
        else:
            failed_cnts = 0
            create_date = task.project.create_date.replace(microsecond=0)
            waited = elapsed = None
        cost = build_cost()
        status = format_status(task.aasm_state, task.aasm_state == 'cleaned')
        row = [task.id, os.path.basename(task.shell), status, failed_cnts, task.module.name, task.app.name, task.instance.name, create_date, waited, elapsed] + cost
        tbl.add_row(row)
    return tbl

def get_date(date):
    if date:
        return date.replace(microsecond=0)
    elif date is None:
        return None
    else:
        raise ValueError("Invalid datetime")

def diff_date(t1, t2):
    if all([t1, t2]):
        return t2 - t1
    elif t1 and not t2:
        now = datetime.datetime.now().replace(microsecond=0)
        return now - t1
    else:
        return None

def format_detail_task(task):
    tbl = PrettyTable()
    tbl.header = False
    fields = ['id', 'name', 'status', 'module', 'app', 'instance', 'docker_image', 'cpu', 'mem', 'disk_type', 'disk_size', 'debug_mode', 'benchmark']
    status = format_status(task.aasm_state, task.aasm_state == 'cleaned')
    values = [task.id, os.path.basename(task.shell), status, task.module.name, task.app.name, task.instance.name, task.docker_image, task.cpu, task.mem, task.disk_type, task.disk_size, task.debug_mode, task.benchmark]
    tbl.add_column("field", fields)
    tbl.add_column("value", values)
    return tbl

def format_bcs_tbl(bcs, with_instance):
    tbl = PrettyTable()
    fields = ['id', 'name', 'status', 'create', 'waited', 'elapsed', 'cost']
    if with_instance:
        fields = fields + ['instance', 'cpu', 'mem', 'instance price', 'spot price']
    tbl.field_names = fields
    for b in bcs:
        status = format_status(b.status.lower(), b.deleted)
        create_date = get_date(b.create_date)
        start_date = get_date(b.start_date)
        finish_date = get_date(b.finish_date)
        waited = diff_date(create_date, start_date)
        elapsed = diff_date(start_date, finish_date)
        row = [b.id, b.name, status, create_date, waited, elapsed, b.cost]
        if with_instance:
            i = b.instance
            row = row + [i.name, i.cpu, i.mem, i.price, b.spot_price_limit]
        tbl.add_row(row)
    return tbl

def format_status(status, deleted):
    if status == 'failed':
        status = dyeFAIL(status)
    elif status == 'finished':
        status = dyeOKGREEN(status)
    elif status == 'stopped':
        status = dyeWARNING(status)
    else:
        status = dyeOKBLUE(status)

    if deleted:
        status = status + " (D)"
    return status

def format_mapping_tbl(mappings, size=False):
    def dye_path(is_exists, path):
        if is_exists:
            return dyeOKGREEN(path)
        else:
            return dyeFAIL(path)

    tbl = PrettyTable()
    fields = ['id', 'name', 'source', 'destination', 'is_write', 'is_immediate', 'is_required']
    if size:
        fields += ['size']
    tbl.field_names = fields
    tbl.align['source'] = "l"
    tbl.align['destination'] = "l"
    tbl.align['size'] = "r"
    for m in mappings:
        source = dye_path(os.path.exists(m.source), m.source)
        destination = dye_path(m.exists(), m.destination)
        row = [m.id, m.name, source, destination, m.is_write, m.is_immediate, m.is_required]
        if size:
            row += [human_size(m.size())]
        tbl.add_row(row)

    return tbl

def format_detail_mapping(mapping, size=False):
    tbl = PrettyTable()
    tbl.header = False
    fields = ['id', 'name', 'source', 'destination', 'is_write', 'is_immediate', 'is_required']
    values = [mapping.__getattribute__(k) for k in fields]

    if mapping.exists():
        values[3] = dyeOKGREEN(values[3])
    else:
        values[3] = dyeFAIL(values[3])

    if size:
        fields += ['size']
        values += [human_size(mapping.size())]

    tbl.add_column("field", fields)
    tbl.add_column("value", values)
    return tbl

def format_instance_tbl(instances, latest_price = False):
    tbl = PrettyTable()
    fields = ['name', 'cpu', 'mem', 'disk type', 'disk size', 'price']
    if latest_price:
        fields += ['spot price', 'origin price', 'discount']
    tbl.field_names = fields
    for i in instances:
        row = [i.name, i.cpu, i.mem, i.disk_type, i.disk_size, i.price]
        if latest_price:
            (spot_price, origin_price) = i.latest_price()
            if origin_price:
                discount = spot_price / origin_price
            else:
                discount = None
            row += [spot_price, origin_price, discount]
        tbl.add_row(row)

    return tbl

def format_cost_tbl(costs):
    tbl = PrettyTable()
    tbl.field_names = ['id', 'name', 'size', 'size cost', 'bcs cost', 'total']
    map(tbl.add_row, costs)
    return tbl

def format_history_price_tbl(prices):
    tbl = PrettyTable()
    tbl.field_names = ['InstanceType', 'Timestamp', 'ZoneId', 'IoOptimized', 'NetworkType', 'OriginPrice', 'SpotPrice', 'Discount']
    for p in prices:
        p['Discount'] = round(p['SpotPrice'] / p['OriginPrice'], 2)
        tbl.add_row([p[key] for key in tbl.field_names])

    return tbl
<|endoftext|>


<commit_before>import datetime, json, jsonpickle
from app import db
from sqlalchemy import Table, Column, Integer, ForeignKey
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base

Base=declarative_base()

class Visits(db.Model):
    __tablename__ = 'results'

    id = db.Column(db.Integer, primary_key=True)
    url = db.Column(db.String())
    timestamp = db.Column(db.DateTime())

    def __init__(self, url):
        self.url = url
        self.timestamp = datetime.datetime.now()

    def __repr__(self):
        return 'id {}'.format(self.id)

class Tag_Product_association(db.Model):
    __tablename__ = 'tag_prod_association'
    product_id = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    tag_name= db.Column(db.String, db.ForeignKey('tag.name'), primary_key=True)

class Product_alternative(db.Model):
    __tablename__ = 'prod_alternatives'
    product_id_1 = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    product_id_2 = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)

class Co2_Product_association(db.Model):
    __tablename__ = 'co2_prod_association'
    product_id = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    co2_id= db.Column(db.Integer, db.ForeignKey('co2.id'), primary_key=True)

class Location_Product_association(db.Model):
    __tablename__ = 'location_prod_association'
    product_id = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    location_id= db.Column(db.Integer, db.ForeignKey('location.id'))

class Location(db.Model):
    __tablename__='location'
    id=db.Column(db.Integer, primary_key=True)
    name=db.Column(db.String)
    possibleOrigins=relationship("Product", secondary="location_prod_association")

class FoodWasteData(db.Model):
    __tablename__='foodwaste'
    id=db.Column(Integer, primary_key=True)
    product_id = Column(Integer, ForeignKey('product.id'))
    product = relationship("Product", back_populates="foodWastes")

class Product(db.Model):
    __tablename__='product'

    def toDict(self):
        return {'id': self.id, 'name': self.name, 'specification': self.specification}

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String())
    specification = db.Column(db.String())
    # synonyms=relationship("Synonym", secondary="synonym_prod_association")
    englishName=db.Column(db.String())
    frenchName=db.Column(db.String())
    Co2Values=relationship("Co2Value")
    tags = relationship("Tag", secondary="tag_prod_association")

    # #Advanced
    alternatives=relationship("Product", secondary="prod_alternatives", primaryjoin=id==Product_alternative.product_id_1,
                           secondaryjoin=id==Product_alternative.product_id_2)
    standardOrigin_id = db.Column(db.Integer, db.ForeignKey('location.id'))
    standardOrigin=relationship(Location, foreign_keys=standardOrigin_id)
    possibleOrigins=relationship("Location", secondary="location_prod_association")
    # productionMethods
    # productionMethodParameters
    # degreesOfProcessing
    # degreesOfProcessingParameters
    # preservationMethods
    # packaging
    # packagingParameters
    # packagingMethods
    startOfLocalSeason=db.Column(db.Date)
    endOfLocalSeason=db.Column(db.Date)
    # density
    # unitWeight
    # commentsOnDensityAndUnitWeight
    # referencesOndensityAndUnitWeight
    # Texture
    foodWastes=relationship("FoodWasteData")
    # CommentsOnFoodwaste
    # Allergenes

    # #Documentation
    # CO2CalculationPath
    # CalculationProcessDocumentation
    # InfoTextForCook
    # ReferencesForBasicCO2Value
    # OtherReferences
    # CommentsOnFoodwasteCO2CalculationPathForDifferentProductParameters
    # DataQualityEstimation

# class FoodWasteCollection(db.Model):
#     __tablename__ = 'foodwaste'
#     product=db.Column(db.ForeignKey)
#     productionAvoidable = db.Column(db.integer())

class Tag(db.Model):
    __tablename__='tag'

    name=db.Column(db.String, primary_key=True)

class Co2Value(db.Model):
    __tablename__='co2'
    id=db.Column(db.Integer, primary_key=True)
    value=db.Column(db.String)
    product_id = Column(Integer, ForeignKey('product.id'))
    product = relationship("Product", back_populates="Co2Values")

# class CO2ValueDerived(Co2Value):

# class CO2ValueBase(Co2Value):

# class Reference(db.Model)
#     __tablename__ = 'reference'

<commit_msg>added classes to model<commit_after>import datetime, json, jsonpickle
from app import db
from sqlalchemy import Table, Column, Integer, ForeignKey
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base

Base=declarative_base()

class Visits(db.Model):
    __tablename__ = 'results'

    id = db.Column(db.Integer, primary_key=True)
    url = db.Column(db.String())
    timestamp = db.Column(db.DateTime())

    def __init__(self, url):
        self.url = url
        self.timestamp = datetime.datetime.now()

    def __repr__(self):
        return 'id {}'.format(self.id)

class TagProductAssociation(db.Model):
    __tablename__ = 'tag_prod_association'
    product_id = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    tag_name= db.Column(db.String, db.ForeignKey('tag.name'), primary_key=True)

class ProductAllergeneAssociation(db.Model):
    __tablename__ = 'prod_allergene_association'
    product_id = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    allergene_id= db.Column(db.Interger, db.ForeignKey('allergene.id'), primary_key=True)
    product=relationship("Product", back_populates="allergenes")
    valueBase_id = db.Column(db.Integer, db.ForeignKey('value_base.id'))
    valueBase=relationship('ValueBase')

class ProductNutrientAssociation(db.Model):
    __tablename__ = 'prod_nutrient_association'
    product_id = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    allergene_id= db.Column(db.Interger, db.ForeignKey('allergene.id'), primary_key=True)
    product=relationship("Product", back_populates="nutrients")
    valueBase_id = db.Column(db.Integer, db.ForeignKey('value_base.id'))
    valueBase=relationship('ValueBase')

class ProductAlternative(db.Model):
    __tablename__ = 'prod_alternatives'
    product_id_1 = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    product_id_2 = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)

class LocationProductAssociation(db.Model):
    __tablename__ = 'location_prod_association'
    product_id = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    location_id= db.Column(db.Integer, db.ForeignKey('location.id'), primary_key=True)

class LocationValueAssociation(db.Model): #possible origins
    __tablename__ = 'location_valule_association'
    product_id = db.Column(db.Integer, db.ForeignKey('product.id'), primary_key=True)
    location_id= db.Column(db.Integer, db.ForeignKey('value_base.id'), primary_key=True)

class Location(db.Model):
    __tablename__='location'
    id=db.Column(db.Integer, primary_key=True)
    name=db.Column(db.String)
    possibleProducts=relationship("Product", secondary="location_prod_association", back_populates="possibleLocations")

class FoodWasteData(db.Model):
    __tablename__='foodwaste'
    id=db.Column(Integer, primary_key=True)
    product_id = Column(Integer, ForeignKey('product.id'))
    product = relationship("Product", back_populates="foodWastes")

class Product(db.Model):
    __tablename__='product'

    def toDict(self):
        return {'id': self.id, 'name': self.name, 'specification': self.specification}

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String())
    specification = db.Column(db.String())
    # synonyms=relationship("Synonym", secondary="synonym_prod_association")
    englishName=db.Column(db.String())
    frenchName=db.Column(db.String())
    Co2Value=relationship("Co2Value", uselist=False, back_populates="product")
    tags = relationship("Tag", secondary="tag_prod_association")
    nutrients=relationship("ProductNutrientAssociation", back_populates=product)
    allergenes=relationship("ProductAllergeneAssociation", back_populates=product)
    # #Advanced
    alternatives=relationship("Product", secondary="prod_alternatives", primaryjoin=id==ProductAlternative.product_id_1,
                           secondaryjoin=id==ProductAlternative.product_id_2)
    standardOrigin_id = db.Column(db.Integer, db.ForeignKey('location.id'))
    standardOrigin=relationship(Location, foreign_keys=standardOrigin_id)
    possibleLocations=relationship("Location", secondary="location_prod_association", back_populates="possibleProducts")
    # productionMethods
    # productionMethodParameters
    # degreesOfProcessing
    # degreesOfProcessingParameters
    # preservationMethods
    # packaging
    # packagingParameters
    # packagingMethods
    startOfLocalSeason=db.Column(db.Date)
    endOfLocalSeason=db.Column(db.Date)
    # density
    # unitWeight
    # commentsOnDensityAndUnitWeight
    # referencesOndensityAndUnitWeight
    # texture
    foodWasteData=relationship("FoodWasteData", back_populates="product")
    # commentsOnFoodwaste

    # #Documentation
    # co2CalculationPath
    # calculationProcessDocumentation
    # infoTextForCook
    # referencesForBasicCO2Value
    # otherReferences
    # commentsOnFoodwasteCO2CalculationPathForDifferentProductParameters
    # dataQualityEstimation

class Process(db.Model):
    __tablename__='process'
    id=db.Column(db.Integer, primary_key=True)

class Allergene(db.Model):#single allergene, not multiple allergenes
    __tablename__='allergene'
    id=db.Column(db.Integer, primary_key=True)
    name=db.Column(db.String)
    shortname=db.Column(db.String)

class Nutrient(db.Model):
    __tablename__='nutrient'
    id=db.Column(db.Integer, primary_key=True)
    name=db.Column(db.String)
    shortname=db.Column(db.String)

class Tag(db.Model):
    __tablename__='tag'
    name=db.Column(db.String, primary_key=True)

class ValueBase(db.Model):
    __tablename__='value_base'
    id=db.Column(db.Integer, primary_key=True)
    validCountries = relationship("Location", secondary="LocationValueAssociation")

class Co2Value(db.Model):
    __tablename__='co2'
    id=db.Column(db.Integer, primary_key=True)
    value=db.Column(db.String)
    valueBase_id = db.Column(db.Integer, db.ForeignKey('value_base.id'))
    product_id = Column(Integer, ForeignKey('product.id'))
    product = relationship("Product", back_populates="Co2Value")

class Reference(db.Model):
    __tablename__ = 'reference'

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2016 Matt Martz
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import sys

try:
    from urllib import urlencode
except ImportError:
    from urllib.parse import urlencode

from flask import Flask

try:
    from cStringIO import StringIO
except ImportError:
    try:
        from StringIO import StringIO
    except ImportError:
        from io import StringIO

from werkzeug.wrappers import BaseRequest

__version__ = '0.0.4'

def make_environ(event):
    environ = {}

    for hdr_name, hdr_value in event['headers'].items():
        hdr_name = hdr_name.replace('-', '_').upper()
        if hdr_name in ['CONTENT_TYPE', 'CONTENT_LENGTH']:
            environ[hdr_name] = hdr_value
            continue

        http_hdr_name = 'HTTP_%s' % hdr_name
        environ[http_hdr_name] = hdr_value

    qs = event['queryStringParameters']

    environ['REQUEST_METHOD'] = event['httpMethod']
    environ['PATH_INFO'] = event['path']
    environ['QUERY_STRING'] = urlencode(qs) if qs else ''
    environ['REMOTE_ADDR'] = event['requestContext']['identity']['sourceIp']
    environ['HOST'] = '%(HTTP_HOST)s:%(HTTP_X_FORWARDED_PORT)s' % environ
    environ['SCRIPT_NAME'] = ''

    environ['SERVER_PORT'] = environ['HTTP_X_FORWARDED_PORT']
    environ['SERVER_PROTOCOL'] = 'HTTP/1.1'

    environ['CONTENT_LENGTH'] = str(
        len(event['body']) if event['body'] else ''
    )

    environ['wsgi.url_scheme'] = environ['HTTP_X_FORWARDED_PROTO']
    environ['wsgi.input'] = StringIO(event['body'] or '')
    environ['wsgi.version'] = (1, 0)
    environ['wsgi.errors'] = sys.stderr
    environ['wsgi.multithread'] = False
    environ['wsgi.run_once'] = True
    environ['wsgi.multiprocess'] = False

    BaseRequest(environ)

    return environ

class LambdaResponse(object):
    def __init__(self):
        self.status = None
        self.response_headers = None

    def start_response(self, status, response_headers, exc_info=None):
        self.status = int(status[:3])
        self.response_headers = dict(response_headers)

class FlaskLambda(Flask):
    def __call__(self, event, context):
        if 'httpMethod' not in event:
            # In this "context" `event` is `environ` and
            # `context` is `start_response`, meaning the request didn't
            # occur via API Gateway and Lambda
            return super(FlaskLambda, self).__call__(event, context)

        response = LambdaResponse()

        body = next(self.wsgi_app(
            make_environ(event),
            response.start_response
        ))

        return {
            'statusCode': response.status,
            'headers': response.response_headers,
            'body': body
        }
<commit_msg>fix: werkzeug 2.1.X support (#14)<commit_after># -*- coding: utf-8 -*-
# Copyright 2016 Matt Martz
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import sys

try:
    from urllib import urlencode
except ImportError:
    from urllib.parse import urlencode

from flask import Flask

try:
    from cStringIO import StringIO
except ImportError:
    try:
        from StringIO import StringIO
    except ImportError:
        from io import StringIO

try: # werkzeug <= 2.0.3
    from werkzeug.wrappers import BaseRequest
except: # werkzeug > 2.1
    from werkzeug.wrappers import Request as BaseRequest

__version__ = '0.0.4'

def make_environ(event):
    environ = {}

    for hdr_name, hdr_value in event['headers'].items():
        hdr_name = hdr_name.replace('-', '_').upper()
        if hdr_name in ['CONTENT_TYPE', 'CONTENT_LENGTH']:
            environ[hdr_name] = hdr_value
            continue

        http_hdr_name = 'HTTP_%s' % hdr_name
        environ[http_hdr_name] = hdr_value

    qs = event['queryStringParameters']

    environ['REQUEST_METHOD'] = event['httpMethod']
    environ['PATH_INFO'] = event['path']
    environ['QUERY_STRING'] = urlencode(qs) if qs else ''
    environ['REMOTE_ADDR'] = event['requestContext']['identity']['sourceIp']
    environ['HOST'] = '%(HTTP_HOST)s:%(HTTP_X_FORWARDED_PORT)s' % environ
    environ['SCRIPT_NAME'] = ''

    environ['SERVER_PORT'] = environ['HTTP_X_FORWARDED_PORT']
    environ['SERVER_PROTOCOL'] = 'HTTP/1.1'

    environ['CONTENT_LENGTH'] = str(
        len(event['body']) if event['body'] else ''
    )

    environ['wsgi.url_scheme'] = environ['HTTP_X_FORWARDED_PROTO']
    environ['wsgi.input'] = StringIO(event['body'] or '')
    environ['wsgi.version'] = (1, 0)
    environ['wsgi.errors'] = sys.stderr
    environ['wsgi.multithread'] = False
    environ['wsgi.run_once'] = True
    environ['wsgi.multiprocess'] = False

    BaseRequest(environ)

    return environ

class LambdaResponse(object):
    def __init__(self):
        self.status = None
        self.response_headers = None

    def start_response(self, status, response_headers, exc_info=None):
        self.status = int(status[:3])
        self.response_headers = dict(response_headers)

class FlaskLambda(Flask):
    def __call__(self, event, context):
        if 'httpMethod' not in event:
            # In this "context" `event` is `environ` and
            # `context` is `start_response`, meaning the request didn't
            # occur via API Gateway and Lambda
            return super(FlaskLambda, self).__call__(event, context)

        response = LambdaResponse()

        body = next(self.wsgi_app(
            make_environ(event),
            response.start_response
        ))

        return {
            'statusCode': response.status,
            'headers': response.response_headers,
            'body': body
        }
<|endoftext|>