<commit_before>import numpy as np
from scipy import sparse as sp
from matutils import mkvc, ndgrid, sub2ind, sdiag
from codeutils import asArray_N_x_Dim
from codeutils import isScalar
import os

def exampleLrmGrid(nC, exType):
    assert type(nC) == list, "nC must be a list containing the number of nodes"
    assert len(nC) == 2 or len(nC) == 3, "nC must either two or three dimensions"
    exType = exType.lower()

    possibleTypes = ['rect', 'rotate']
    assert exType in possibleTypes, "Not a possible example type."

    if exType == 'rect':
        return list(ndgrid([np.cumsum(np.r_[0, np.ones(nx)/nx]) for nx in nC], vector=False))
    elif exType == 'rotate':
        if len(nC) == 2:
            X, Y = ndgrid([np.cumsum(np.r_[0, np.ones(nx)/nx]) for nx in nC], vector=False)
            amt = 0.5-np.sqrt((X - 0.5)**2 + (Y - 0.5)**2)
            amt[amt < 0] = 0
            return [X + (-(Y - 0.5))*amt, Y + (+(X - 0.5))*amt]
        elif len(nC) == 3:
            X, Y, Z = ndgrid([np.cumsum(np.r_[0, np.ones(nx)/nx]) for nx in nC], vector=False)
            amt = 0.5-np.sqrt((X - 0.5)**2 + (Y - 0.5)**2 + (Z - 0.5)**2)
            amt[amt < 0] = 0
            return [X + (-(Y - 0.5))*amt, Y + (-(Z - 0.5))*amt, Z + (-(X - 0.5))*amt]

def meshTensor(value):
    """
        **meshTensor** takes a list of numbers and tuples that have the form::

            mT = [ float, (cellSize, numCell), (cellSize, numCell, factor) ]

        For example, a time domain mesh code needs
        many time steps at one time::

            [(1e-5, 30), (1e-4, 30), 1e-3]

        Means take 30 steps at 1e-5 and then 30 more at 1e-4,
        and then one step of 1e-3.

        Tensor meshes can also be created by increase factors::

            [(10.0, 5, -1.3), (10.0, 50), (10.0, 5, 1.3)]

        When there is a third number in the tuple, it
        refers to the increase factor, if this number
        is negative this section of the tensor is flipped right-to-left.

        .. plot::

            from SimPEG import Mesh
            tx = [(10.0,10,-1.3),(10.0,40),(10.0,10,1.3)]
            ty = [(10.0,10,-1.3),(10.0,40)]
            M = Mesh.TensorMesh([tx, ty])
            M.plotGrid(showIt=True)

    """
    if type(value) is not list:
        raise Exception('meshTensor must be a list of scalars and tuples.')

    proposed = []
    for v in value:
        if isScalar(v):
            proposed += [float(v)]
        elif type(v) is tuple and len(v) == 2:
            proposed += [float(v[0])]*int(v[1])
        elif type(v) is tuple and len(v) == 3:
            start = float(v[0])
            num = int(v[1])
            factor = float(v[2])
            pad = ((np.ones(num)*np.abs(factor))**(np.arange(num)+1))*start
            if factor < 0: pad = pad[::-1]
            proposed += pad.tolist()
        else:
            raise Exception('meshTensor must contain only scalars and len(2) or len(3) tuples.')

    return np.array(proposed)

def closestPoints(mesh, pts, gridLoc='CC'):
    """
        Move a list of points to the closest points on a grid.

        :param simpeg.Mesh.BaseMesh mesh: The mesh
        :param numpy.ndarray pts: Points to move
        :param string gridLoc: ['CC', 'N', 'Fx', 'Fy', 'Fz', 'Ex', 'Ex', 'Ey', 'Ez']
        :rtype: numpy.ndarray
        :return: nodeInds
    """

    pts = asArray_N_x_Dim(pts, mesh.dim)
    grid = getattr(mesh, 'grid' + gridLoc)
    nodeInds = np.empty(pts.shape[0], dtype=int)

    for i, pt in enumerate(pts):
        if mesh.dim == 1:
            nodeInds[i] = ((pt - grid)**2).argmin()
        else:
            nodeInds[i] = ((np.tile(pt, (grid.shape[0],1)) - grid)**2).sum(axis=1).argmin()

    return nodeInds

def readUBCTensorMesh(fileName):
    """
        Read UBC GIF 3DTensor mesh and generate 3D Tensor mesh in simpegTD

        Input:
        :param fileName, path to the UBC GIF mesh file

        Output:
        :param SimPEG TensorMesh object
        :return
    """

    # Interal function to read cell size lines for the UBC mesh files.
    def readCellLine(line):
        for seg in line.split():
            if '*' in seg:
                st = seg
                sp = seg.split('*')
                re = np.array(sp[0],dtype=int)*(' ' + sp[1])
                line = line.replace(st,re.strip())
        return np.array(line.split(),dtype=float)

    # Read the file as line strings, remove lines with comment = !
    msh = np.genfromtxt(fileName,delimiter='\n',dtype=np.str,comments='!')

    # Fist line is the size of the model
    sizeM = np.array(msh[0].split(),dtype=float)
    # Second line is the South-West-Top corner coordinates.
    x0 = np.array(msh[1].split(),dtype=float)
    # Read the cell sizes
    h1 = readCellLine(msh[2])
    h2 = readCellLine(msh[3])
    h3temp = readCellLine(msh[4])
    h3 = h3temp[::-1] # Invert the indexing of the vector to start from the bottom.
    # Adjust the reference point to the bottom south west corner
    x0[2] = x0[2] - np.sum(h3)
    # Make the mesh
    from SimPEG import Mesh
    tensMsh = Mesh.TensorMesh([h1,h2,h3],x0)
    return tensMsh

def readUBCTensorModel(fileName, mesh):
    """
        ReadUBC 3DTensor mesh model and generate 3D Tensor mesh model in simpegTD

    """
    f = open(fileName, 'r')
    model = np.array(map(float, f.readlines()))
    f.close()
    model = np.reshape(model, (mesh.nCz, mesh.nCx, mesh.nCy), order = 'F')
    model = model[::-1,:,:]
    model = np.transpose(model, (1, 2, 0))
    model = mkvc(model)

    return model

def writeUBCTensorMesh(fileName, mesh):
    """
        Writes a SimPEG TensorMesh to a UBC-GIF format mesh file.

        :param str fileName: File to write to
        :param simpeg.Mesh.TensorMesh mesh: The mesh

    """
    assert mesh.dim == 3
    s = ''
    s += '%i %i %i\n' %tuple(mesh.vnC)
    origin = mesh.x0 + np.array([0,0,mesh.hz.sum()]) # Have to it in the same operation or use mesh.x0.copy(), otherwise the mesh.x0 is updated.
    origin.dtype = float

    s += '%.2f %.2f %.2f\n' %tuple(origin)
    s += ('%.2f '*mesh.nCx+'\n')%tuple(mesh.hx)
    s += ('%.2f '*mesh.nCy+'\n')%tuple(mesh.hy)
    s += ('%.2f '*mesh.nCz+'\n')%tuple(mesh.hz[::-1])
    f = open(fileName, 'w')
    f.write(s)
    f.close()

def writeUBCTensorModel(fileName, mesh, model):
    """
        Writes a model associated with a SimPEG TensorMesh
        to a UBC-GIF format model file.

        :param str fileName: File to write to
        :param simpeg.Mesh.TensorMesh mesh: The mesh
        :param numpy.ndarray model: The model
    """

    # Reshape model to a matrix
    modelMat = mesh.r(model,'CC','CC','M')
    # Transpose the axes
    modelMatT = modelMat.transpose((2,0,1))
    # Flip z to positive down
    modelMatTR = mkvc(modelMatT[::-1,:,:])

    np.savetxt(fileName, modelMatTR.ravel())

def readVTRFile(fileName):
    """
        Read VTK Rectilinear (vtr xml file) and return SimPEG Tensor mesh and model

        Input:
        :param vtrFileName, path to the vtr model file

        Output:
        :param SimPEG TensorMesh object
        :param SimPEG model dictionary
        :return
    """
    # Import
    from vtk import vtkXMLRectilinearGridReader as vtrFileReader
    from vtk.util.numpy_support import vtk_to_numpy

    # Read the file
    vtrReader = vtrFileReader()
    vtrReader.SetFileName(fileName)
    vtrReader.Update()
    vtrGrid = vtrReader.GetOutput()
    # Sort information
    hx = np.abs(np.diff(vtk_to_numpy(vtrGrid.GetXCoordinates())))
    xR = vtk_to_numpy(vtrGrid.GetXCoordinates())[0]
    hy = np.abs(np.diff(vtk_to_numpy(vtrGrid.GetYCoordinates())))
    yR = vtk_to_numpy(vtrGrid.GetYCoordinates())[0]
    zD = np.diff(vtk_to_numpy(vtrGrid.GetZCoordinates()))
    # Check the direction of hz
    if np.all(zD < 0):
        hz = np.abs(zD[::-1])
        zR = vtk_to_numpy(vtrGrid.GetZCoordinates())[-1]
    else:
        hz = np.abs(zD)
        zR = vtk_to_numpy(vtrGrid.GetZCoordinates())[0]
    x0 = np.array([xR,yR,zR])

    # Make the SimPEG object
    from SimPEG import Mesh
    tensMsh = Mesh.TensorMesh([hx,hy,hz],x0)

    # Grap the models
    modelDict = {}
    for i in np.arange(vtrGrid.GetCellData().GetNumberOfArrays()):
        modelName = vtrGrid.GetCellData().GetArrayName(i)
        if np.all(zD < 0):
            modFlip = vtk_to_numpy(vtrGrid.GetCellData().GetArray(i))
            tM = tensMsh.r(modFlip,'CC','CC','M')
            modArr = tensMsh.r(tM[:,:,::-1],'CC','CC','V')
        else:
            modArr = vtk_to_numpy(vtrGrid.GetCellData().GetArray(i))
        modelDict[modelName] = modArr

    # Return the data
    return tensMsh, modelDict

def writeVTRFile(fileName,mesh,model=None):
    """
    Makes and saves a VTK rectilinear file (vtr) for a simpeg Tensor mesh and model.

    Input:
    :param str, path to the output vtk file
    :param mesh, SimPEG TensorMesh object - mesh to be transfer to VTK
    :param model, dictionary of numpy.array - Name('s) and array('s). Match number of cells

    """
    # Import
    from vtk import vtkRectilinearGrid as rectGrid, vtkXMLRectilinearGridWriter as rectWriter
    from vtk.util.numpy_support import numpy_to_vtk

    # Deal with dimensionalities
    if mesh.dim >= 1:
        vX = mesh.vectorNx
        xD = mesh.nNx
        yD,zD = 1,1
        vY, vZ = np.array([0,0])
    if mesh.dim >= 2:
        vY = mesh.vectorNy
        yD = mesh.nNy
    if mesh.dim == 3:
        vZ = mesh.vectorNz
        zD = mesh.nNz
    # Use rectilinear VTK grid.
    # Assign the spatial information.
    vtkObj = rectGrid()
    vtkObj.SetDimensions(xD,yD,zD)
    vtkObj.SetXCoordinates(numpy_to_vtk(vX,deep=1))
    vtkObj.SetYCoordinates(numpy_to_vtk(vY,deep=1))
    vtkObj.SetZCoordinates(numpy_to_vtk(vZ,deep=1))

    # Assign the model('s) to the object
    for item in model.iteritems():
        # Convert numpy array
        vtkDoubleArr = numpy_to_vtk(item[1],deep=1)
        vtkDoubleArr.SetName(item[0])
        vtkObj.GetCellData().AddArray(vtkDoubleArr)
    # Set the active scalar
    vtkObj.GetCellData().SetActiveScalars(model.keys()[0])
    vtkObj.Update()

    # Check the extension of the fileName
    ext = os.path.splitext(fileName)[1]
    if ext is '':
        fileName = fileName + '.vtr'
    elif ext not in '.vtr':
        raise IOError('{:s} is an incorrect extension, has to be .vtr')
    # Write the file.
    vtrWriteFilter = rectWriter()
    vtrWriteFilter.SetInput(vtkObj)
    vtrWriteFilter.SetFileName(fileName)
    vtrWriteFilter.Update()

if __name__ == '__main__':
    from SimPEG import Mesh
    import matplotlib.pyplot as plt
    tx = [(10.0,10,-1.3),(10.0,40),(10.0,10,1.3)]
    ty = [(10.0,10,-1.3),(10.0,40)]
    M = Mesh.TensorMesh([tx, ty])
    M.plotGrid()
    plt.gca().axis('tight')
    plt.show()
<commit_msg>Updated the function help text<commit_after>import numpy as np
from scipy import sparse as sp
from matutils import mkvc, ndgrid, sub2ind, sdiag
from codeutils import asArray_N_x_Dim
from codeutils import isScalar
import os

def exampleLrmGrid(nC, exType):
    assert type(nC) == list, "nC must be a list containing the number of nodes"
    assert len(nC) == 2 or len(nC) == 3, "nC must either two or three dimensions"
    exType = exType.lower()

    possibleTypes = ['rect', 'rotate']
    assert exType in possibleTypes, "Not a possible example type."

    if exType == 'rect':
        return list(ndgrid([np.cumsum(np.r_[0, np.ones(nx)/nx]) for nx in nC], vector=False))
    elif exType == 'rotate':
        if len(nC) == 2:
            X, Y = ndgrid([np.cumsum(np.r_[0, np.ones(nx)/nx]) for nx in nC], vector=False)
            amt = 0.5-np.sqrt((X - 0.5)**2 + (Y - 0.5)**2)
            amt[amt < 0] = 0
            return [X + (-(Y - 0.5))*amt, Y + (+(X - 0.5))*amt]
        elif len(nC) == 3:
            X, Y, Z = ndgrid([np.cumsum(np.r_[0, np.ones(nx)/nx]) for nx in nC], vector=False)
            amt = 0.5-np.sqrt((X - 0.5)**2 + (Y - 0.5)**2 + (Z - 0.5)**2)
            amt[amt < 0] = 0
            return [X + (-(Y - 0.5))*amt, Y + (-(Z - 0.5))*amt, Z + (-(X - 0.5))*amt]

def meshTensor(value):
    """
        **meshTensor** takes a list of numbers and tuples that have the form::

            mT = [ float, (cellSize, numCell), (cellSize, numCell, factor) ]

        For example, a time domain mesh code needs
        many time steps at one time::

            [(1e-5, 30), (1e-4, 30), 1e-3]

        Means take 30 steps at 1e-5 and then 30 more at 1e-4,
        and then one step of 1e-3.

        Tensor meshes can also be created by increase factors::

            [(10.0, 5, -1.3), (10.0, 50), (10.0, 5, 1.3)]

        When there is a third number in the tuple, it
        refers to the increase factor, if this number
        is negative this section of the tensor is flipped right-to-left.

        .. plot::

            from SimPEG import Mesh
            tx = [(10.0,10,-1.3),(10.0,40),(10.0,10,1.3)]
            ty = [(10.0,10,-1.3),(10.0,40)]
            M = Mesh.TensorMesh([tx, ty])
            M.plotGrid(showIt=True)

    """
    if type(value) is not list:
        raise Exception('meshTensor must be a list of scalars and tuples.')

    proposed = []
    for v in value:
        if isScalar(v):
            proposed += [float(v)]
        elif type(v) is tuple and len(v) == 2:
            proposed += [float(v[0])]*int(v[1])
        elif type(v) is tuple and len(v) == 3:
            start = float(v[0])
            num = int(v[1])
            factor = float(v[2])
            pad = ((np.ones(num)*np.abs(factor))**(np.arange(num)+1))*start
            if factor < 0: pad = pad[::-1]
            proposed += pad.tolist()
        else:
            raise Exception('meshTensor must contain only scalars and len(2) or len(3) tuples.')

    return np.array(proposed)

def closestPoints(mesh, pts, gridLoc='CC'):
    """
        Move a list of points to the closest points on a grid.

        :param simpeg.Mesh.BaseMesh mesh: The mesh
        :param numpy.ndarray pts: Points to move
        :param string gridLoc: ['CC', 'N', 'Fx', 'Fy', 'Fz', 'Ex', 'Ex', 'Ey', 'Ez']
        :rtype: numpy.ndarray
        :return: nodeInds
    """

    pts = asArray_N_x_Dim(pts, mesh.dim)
    grid = getattr(mesh, 'grid' + gridLoc)
    nodeInds = np.empty(pts.shape[0], dtype=int)

    for i, pt in enumerate(pts):
        if mesh.dim == 1:
            nodeInds[i] = ((pt - grid)**2).argmin()
        else:
            nodeInds[i] = ((np.tile(pt, (grid.shape[0],1)) - grid)**2).sum(axis=1).argmin()

    return nodeInds

def readUBCTensorMesh(fileName):
    """
        Read UBC GIF 3DTensor mesh and generate 3D Tensor mesh in simpegTD

        Input:
        :param fileName, path to the UBC GIF mesh file

        Output:
        :param SimPEG TensorMesh object
        :return
    """

    # Interal function to read cell size lines for the UBC mesh files.
    def readCellLine(line):
        for seg in line.split():
            if '*' in seg:
                st = seg
                sp = seg.split('*')
                re = np.array(sp[0],dtype=int)*(' ' + sp[1])
                line = line.replace(st,re.strip())
        return np.array(line.split(),dtype=float)

    # Read the file as line strings, remove lines with comment = !
    msh = np.genfromtxt(fileName,delimiter='\n',dtype=np.str,comments='!')

    # Fist line is the size of the model
    sizeM = np.array(msh[0].split(),dtype=float)
    # Second line is the South-West-Top corner coordinates.
    x0 = np.array(msh[1].split(),dtype=float)
    # Read the cell sizes
    h1 = readCellLine(msh[2])
    h2 = readCellLine(msh[3])
    h3temp = readCellLine(msh[4])
    h3 = h3temp[::-1] # Invert the indexing of the vector to start from the bottom.
    # Adjust the reference point to the bottom south west corner
    x0[2] = x0[2] - np.sum(h3)
    # Make the mesh
    from SimPEG import Mesh
    tensMsh = Mesh.TensorMesh([h1,h2,h3],x0)
    return tensMsh

def readUBCTensorModel(fileName, mesh):
    """
        Read UBC 3DTensor mesh model and generate 3D Tensor mesh model in simpeg

        Input:
        :param fileName, path to the UBC GIF mesh file to read
        :param mesh, TensorMesh object, mesh that coresponds to the model

        Output:
        :return numpy array, model with TensorMesh ordered
    """
    f = open(fileName, 'r')
    model = np.array(map(float, f.readlines()))
    f.close()
    model = np.reshape(model, (mesh.nCz, mesh.nCx, mesh.nCy), order = 'F')
    model = model[::-1,:,:]
    model = np.transpose(model, (1, 2, 0))
    model = mkvc(model)

    return model

def writeUBCTensorMesh(fileName, mesh):
    """
        Writes a SimPEG TensorMesh to a UBC-GIF format mesh file.

        :param str fileName: File to write to
        :param simpeg.Mesh.TensorMesh mesh: The mesh

    """
    assert mesh.dim == 3
    s = ''
    s += '%i %i %i\n' %tuple(mesh.vnC)
    origin = mesh.x0 + np.array([0,0,mesh.hz.sum()]) # Have to it in the same operation or use mesh.x0.copy(), otherwise the mesh.x0 is updated.
    origin.dtype = float

    s += '%.2f %.2f %.2f\n' %tuple(origin)
    s += ('%.2f '*mesh.nCx+'\n')%tuple(mesh.hx)
    s += ('%.2f '*mesh.nCy+'\n')%tuple(mesh.hy)
    s += ('%.2f '*mesh.nCz+'\n')%tuple(mesh.hz[::-1])
    f = open(fileName, 'w')
    f.write(s)
    f.close()

def writeUBCTensorModel(fileName, mesh, model):
    """
        Writes a model associated with a SimPEG TensorMesh
        to a UBC-GIF format model file.

        :param str fileName: File to write to
        :param simpeg.Mesh.TensorMesh mesh: The mesh
        :param numpy.ndarray model: The model
    """

    # Reshape model to a matrix
    modelMat = mesh.r(model,'CC','CC','M')
    # Transpose the axes
    modelMatT = modelMat.transpose((2,0,1))
    # Flip z to positive down
    modelMatTR = mkvc(modelMatT[::-1,:,:])

    np.savetxt(fileName, modelMatTR.ravel())

def readVTRFile(fileName):
    """
        Read VTK Rectilinear (vtr xml file) and return SimPEG Tensor mesh and model

        Input:
        :param vtrFileName, path to the vtr model file to write to

        Output:
        :return SimPEG TensorMesh object
        :return SimPEG model dictionary

    """
    # Import
    from vtk import vtkXMLRectilinearGridReader as vtrFileReader
    from vtk.util.numpy_support import vtk_to_numpy

    # Read the file
    vtrReader = vtrFileReader()
    vtrReader.SetFileName(fileName)
    vtrReader.Update()
    vtrGrid = vtrReader.GetOutput()
    # Sort information
    hx = np.abs(np.diff(vtk_to_numpy(vtrGrid.GetXCoordinates())))
    xR = vtk_to_numpy(vtrGrid.GetXCoordinates())[0]
    hy = np.abs(np.diff(vtk_to_numpy(vtrGrid.GetYCoordinates())))
    yR = vtk_to_numpy(vtrGrid.GetYCoordinates())[0]
    zD = np.diff(vtk_to_numpy(vtrGrid.GetZCoordinates()))
    # Check the direction of hz
    if np.all(zD < 0):
        hz = np.abs(zD[::-1])
        zR = vtk_to_numpy(vtrGrid.GetZCoordinates())[-1]
    else:
        hz = np.abs(zD)
        zR = vtk_to_numpy(vtrGrid.GetZCoordinates())[0]
    x0 = np.array([xR,yR,zR])

    # Make the SimPEG object
    from SimPEG import Mesh
    tensMsh = Mesh.TensorMesh([hx,hy,hz],x0)

    # Grap the models
    modelDict = {}
    for i in np.arange(vtrGrid.GetCellData().GetNumberOfArrays()):
        modelName = vtrGrid.GetCellData().GetArrayName(i)
        if np.all(zD < 0):
            modFlip = vtk_to_numpy(vtrGrid.GetCellData().GetArray(i))
            tM = tensMsh.r(modFlip,'CC','CC','M')
            modArr = tensMsh.r(tM[:,:,::-1],'CC','CC','V')
        else:
            modArr = vtk_to_numpy(vtrGrid.GetCellData().GetArray(i))
        modelDict[modelName] = modArr

    # Return the data
    return tensMsh, modelDict

def writeVTRFile(fileName,mesh,model=None):
    """
    Makes and saves a VTK rectilinear file (vtr) for a simpeg Tensor mesh and model.

    Input:
    :param str, path to the output vtk file
    :param mesh, SimPEG TensorMesh object - mesh to be transfer to VTK
    :param model, dictionary of numpy.array - Name('s) and array('s). Match number of cells

    """
    # Import
    from vtk import vtkRectilinearGrid as rectGrid, vtkXMLRectilinearGridWriter as rectWriter
    from vtk.util.numpy_support import numpy_to_vtk

    # Deal with dimensionalities
    if mesh.dim >= 1:
        vX = mesh.vectorNx
        xD = mesh.nNx
        yD,zD = 1,1
        vY, vZ = np.array([0,0])
    if mesh.dim >= 2:
        vY = mesh.vectorNy
        yD = mesh.nNy
    if mesh.dim == 3:
        vZ = mesh.vectorNz
        zD = mesh.nNz
    # Use rectilinear VTK grid.
    # Assign the spatial information.
    vtkObj = rectGrid()
    vtkObj.SetDimensions(xD,yD,zD)
    vtkObj.SetXCoordinates(numpy_to_vtk(vX,deep=1))
    vtkObj.SetYCoordinates(numpy_to_vtk(vY,deep=1))
    vtkObj.SetZCoordinates(numpy_to_vtk(vZ,deep=1))

    # Assign the model('s) to the object
    for item in model.iteritems():
        # Convert numpy array
        vtkDoubleArr = numpy_to_vtk(item[1],deep=1)
        vtkDoubleArr.SetName(item[0])
        vtkObj.GetCellData().AddArray(vtkDoubleArr)
    # Set the active scalar
    vtkObj.GetCellData().SetActiveScalars(model.keys()[0])
    vtkObj.Update()

    # Check the extension of the fileName
    ext = os.path.splitext(fileName)[1]
    if ext is '':
        fileName = fileName + '.vtr'
    elif ext not in '.vtr':
        raise IOError('{:s} is an incorrect extension, has to be .vtr')
    # Write the file.
    vtrWriteFilter = rectWriter()
    vtrWriteFilter.SetInput(vtkObj)
    vtrWriteFilter.SetFileName(fileName)
    vtrWriteFilter.Update()

if __name__ == '__main__':
    from SimPEG import Mesh
    import matplotlib.pyplot as plt
    tx = [(10.0,10,-1.3),(10.0,40),(10.0,10,1.3)]
    ty = [(10.0,10,-1.3),(10.0,40)]
    M = Mesh.TensorMesh([tx, ty])
    M.plotGrid()
    plt.gca().axis('tight')
    plt.show()
<|endoftext|>


<commit_before>"""Call arbitrary API endpoints."""

import click

import json

from SoftLayer.CLI import environment
from SoftLayer.CLI import formatting

def validate_filter(ctx, param, value):
    try:
        if value:
            return json.loads(value)
    except ValueError:
        raise click.BadParameter('filters need to be in JSON format')

@click.command('call', short_help="Call arbitrary API endpoints.")
@click.argument('service')
@click.argument('method')
@click.argument('parameters', nargs=-1)
@click.option('--id', '_id', help="Init parameter")
@click.option('--filter', '_filter', callback=validate_filter, help="Object filter in a JSON string")
@click.option('--mask', help="String-based object mask")
@click.option('--limit', type=click.INT, help="Result limit")
@click.option('--offset', type=click.INT, help="Result offset")
@environment.pass_env
def cli(env, service, method, parameters, _id, _filter, mask, limit, offset):
    """Call arbitrary API endpoints with the given SERVICE and METHOD.

    \b
    Examples:
    slcli call-api Account getObject
    slcli call-api Account getVirtualGuests --limit=10 --mask=id,hostname
    slcli call-api Virtual_Guest getObject --id=12345
    slcli call-api Metric_Tracking_Object getBandwidthData --id=1234 \\
        "2015-01-01 00:00:00" "2015-01-1 12:00:00" public
    """
    result = env.client.call(service, method, *parameters,
                             id=_id,
                             filter=_filter,
                             mask=mask,
                             limit=limit,
                             offset=offset)
    env.fout(formatting.iter_to_table(result))
<commit_msg>Fix style small issue<commit_after>"""Call arbitrary API endpoints."""

import click

import json

from SoftLayer.CLI import environment
from SoftLayer.CLI import formatting

# pylint: disable=unused-argument

def validate_filter(ctx, param, value):
    """Try to parse the given filter as a JSON string."""
    try:
        if value:
            return json.loads(value)
    except ValueError:
        raise click.BadParameter('filters need to be in JSON format')

@click.command('call', short_help="Call arbitrary API endpoints.")
@click.argument('service')
@click.argument('method')
@click.argument('parameters', nargs=-1)
@click.option('--id', '_id', help="Init parameter")
@click.option('--filter', '_filter',
              callback=validate_filter,
              help="Object filter in a JSON string")
@click.option('--mask', help="String-based object mask")
@click.option('--limit', type=click.INT, help="Result limit")
@click.option('--offset', type=click.INT, help="Result offset")
@environment.pass_env
def cli(env, service, method, parameters, _id, _filter, mask, limit, offset):
    """Call arbitrary API endpoints with the given SERVICE and METHOD.

    \b
    Examples:
    slcli call-api Account getObject
    slcli call-api Account getVirtualGuests --limit=10 --mask=id,hostname
    slcli call-api Virtual_Guest getObject --id=12345
    slcli call-api Metric_Tracking_Object getBandwidthData --id=1234 \\
        "2015-01-01 00:00:00" "2015-01-1 12:00:00" public
    """
    result = env.client.call(service, method, *parameters,
                             id=_id,
                             filter=_filter,
                             mask=mask,
                             limit=limit,
                             offset=offset)
    env.fout(formatting.iter_to_table(result))
<|endoftext|>


<commit_before>#!/usr/bin/env python2
"""
This script identifies the max speed of CPUs based on the model name, then monitors the cores for speed stepping.
The goal is to retain the highest speed and identify cores that are not scaling properly.
monitor /proc/cpuinfo and track for changes
"""

import re
from time import sleep
import sys
try:
    import argparse
    ARGPARSE = True
except:
    ARGPARSE = False

class argsAlt(object):
    pass

class AutoVivification(dict):
    """Implementation of perl's autovivification feature."""
    def __getitem__(self, item):
        try:
            return dict.__getitem__(self, item)
        except KeyError:
            value = self[item] = type(self)()
            return value

class ansi:

    """
    This class is to display different color fonts
    example:
    print "Testing a color    [ %sOK%s ]" % (
        ansi.CYAN,
        ansi.ENDC
                    )
    or to avoid a new line
    import sys
    sys.stdout.write("%s%s" % (ansi.CLR,ansi.HOME))
    """
    PURPLE = '\033[95m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    CYAN = '\033[96m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
    CLR = '\033[2J'
    HOME = '\033[H'
    def clear(self):
        sys.stdout.write("%s%s" % (ansi.CLR,ansi.HOME))
    def home(self):
        sys.stdout.write("%s" % ansi.HOME)

def all_clear(cpuspeed):
    allclear = 1
    for key in cpuspeed:
        if cpuspeed[key]["mhz"]==1200:
            allclear=0
        elif not cpuspeed[key]["mhz"] >= cpuspeed[key]["maxmhz"]:
            allclear=0
    return(allclear)

def screen_print(cpuspeed, cycle_counter, **kwargs):
    """
    kwargs:
    clear_screen = True
    Clear the screen instead of just going to home

    no_ansi = True
    Omit ansi characters for home, clear screen and colors

    no_runtime = True
    Do not print "ctrl+c" and seconds runtime

    """
    if "clear_screen" in kwargs:
        sys.stdout.write("%s%s" % (ansi.CLR,ansi.HOME))
    elif not "no_ansi" in kwargs:
        sys.stdout.write("%s" % (ansi.HOME,))
    print "A healthy and busy system should show MHz in increments of 100 (or XX01 at full speed) and hit full speed on all CPUs in 30-60 seconds.\n"
    for key, value in sorted(cpuspeed.items()):
        # "stuck" CPUs tend to stay at 1200MHz, so flag those as red
        # If the CPU hit the model's max, or 1 over max if it is static, flag it green
        # If the CPU is in between, mark it yellow
        print "CPU : %s, Model: %s, Max: %s" % (key,cpuspeed[key]["model"],cpuspeed[key]["maxmhz"])
        if not "no_ansi" in kwargs:
            if cpuspeed[key]["mhz"]==1200:
                status_color=ansi.RED
            elif not cpuspeed[key]["mhz"] >= cpuspeed[key]["maxmhz"]:
                status_color=ansi.YELLOW
            else:
                status_color=ansi.GREEN
        else:
            status_color=""
        print "Max observed speed: %s%s%s" % (status_color,cpuspeed[key]["mhz"],ansi.ENDC)
    if not "no_runtime" in kwargs:
        print "\nCtrl+C to exit, Runtime: %5.1f seconds" % (float(cycle_counter)/10,)
    return(allclear)

if ARGPARSE:
    parser = argparse.ArgumentParser()
    parser.add_argument("-r","--runtime", help="Maximum number of seconds to watch the CPUs before ending",
                        type=int)
    parser.add_argument("-s","--silent",
                        help="No output, return code of 0 if CPUs are clear, and non-zero if the CPUs do not hit max within the runtime specified. Default 30 seconds, use --runtime to specify the duration.",
                        action="store_true")
    parser.add_argument("-b","--batch", help="Run for a time, then output results. Default 30 seconds, use --runtime to specify the duration.",
                        action="store_true")
    parser.add_argument("--plaintext", help="ANSI control characters are omitted for colors and screen clear/home.",
                        action="store_true")
    args = parser.parse_args()
else:
    args = argsAlt()
    # dummy class in the event argparse is not available on the system

if not hasattr(sys, 'ps1') and not sys.flags.interactive:
    # not interactively
    args.batch = True

if (args.silent or args.batch) and not args.runtime:
    args.runtime = 30
    pass
if args.batch:
    args.plaintext = True
if args.batch:
    pass

cpuspeed = AutoVivification()
cycle_counter = 0
allclear = 0

if not (args.silent or args.plaintext):
    sys.stdout.write("%s%s" % (ansi.CLR,ansi.HOME))

try:
    while allclear == 0:
        infile = open('/proc/cpuinfo', 'r')
        cpuinfo = infile.readlines()
        cpu=""
        for line in cpuinfo:
            # cpu
            # model
            # mhz
            result = re.match('(processor|model name|cpu MHz)[\s:]+(\d*)(.*)', line.strip())
            # group 1 is category
            # group 2 is numbers (speed or processor number)
            # group 3 is anything following numbers (model name)

            if not result:
                continue
            if result.group(1)=="processor":
                # which CPU core are we talking about?
                cpu = int(result.group(2))
            elif result.group(1)=="model name" and not "model" in cpuspeed[cpu]:
                # we only set the model if it isn't already set, and we get the max speed at the same time too
                cpuspeed[cpu]["model"]=result.group(3)
                result = re.search('([\d\.]+)GHz', line.strip())
                cpuspeed[cpu]["maxmhz"]=int(float(re.search('([\d\.]+)GHz', line.strip()).group(1))*1000)
            elif result.group(1)=="cpu MHz":
                # you can't reference the variable that isn't set, so max alone doesn't work
                if not "mhz" in cpuspeed[cpu]:
                    cpuspeed[cpu]["mhz"]=int(result.group(2))
                else:
                    cpuspeed[cpu]["mhz"]=max(cpuspeed[cpu]["mhz"],int(result.group(2)))
        # assume all the CPUs are good (allclear), then test for lower states. Exit if they are all clear.
        allclear = all_clear(cpuspeed)
        if not (args.silent or args.batch):
            if args.plaintext:
                screen_print(cpuspeed, cycle_counter, no_ansi = True)
            else:
                screen_print(cpuspeed, cycle_counter)
        if args.runtime and (cycle_counter/10 >= args.runtime):
            allclear = 2
        infile.close()
        cycle_counter+=1
        sleep(.1)
except KeyboardInterrupt:
    #allclear = all_clear(cpuspeed)
    screen_print(cpuspeed, cycle_counter)
    sys.exit(128)

if args.plaintext:
    screen_print(cpuspeed, cycle_counter, no_ansi = True, no_runtime = True)
    pass

if allclear==1:
    if not args.silent:
        print "All CPUs cleared in %d seconds" % (cycle_counter/10)
    else:
        sys.exit(0)
elif allclear==2:
    plural=""
    if cycle_counter/10 > 1:
        plural="s"
    if not args.silent:
        print "CPUs did not clear during the test run, stopped after %d second%s" % ((cycle_counter/10), plural)
    sys.exit(1)<commit_msg>If the python is not running in a terminal, it goes to batch mode automatically (no refresh, and no ANSI colors).<commit_after>#!/usr/bin/env python2
"""
This script identifies the max speed of CPUs based on the model name, then monitors the cores for speed stepping.
The goal is to retain the highest speed and identify cores that are not scaling properly.
monitor /proc/cpuinfo and track for changes
"""

import re
from time import sleep
import sys
try:
    import argparse
    ARGPARSE = True
except:
    ARGPARSE = False

class argsAlt(object):
    pass

class AutoVivification(dict):
    """Implementation of perl's autovivification feature."""
    def __getitem__(self, item):
        try:
            return dict.__getitem__(self, item)
        except KeyError:
            value = self[item] = type(self)()
            return value

class ansi:

    """
    This class is to display different color fonts
    example:
    print "Testing a color    [ %sOK%s ]" % (
        ansi.CYAN,
        ansi.ENDC
                    )
    or to avoid a new line
    import sys
    sys.stdout.write("%s%s" % (ansi.CLR,ansi.HOME))
    """
    PURPLE = '\033[95m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    CYAN = '\033[96m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
    CLR = '\033[2J'
    HOME = '\033[H'
    def clear(self):
        sys.stdout.write("%s%s" % (ansi.CLR,ansi.HOME))
    def home(self):
        sys.stdout.write("%s" % ansi.HOME)

def all_clear(cpuspeed):
    allclear = 1
    for key in cpuspeed:
        if cpuspeed[key]["mhz"]==1200:
            allclear=0
        elif not cpuspeed[key]["mhz"] >= cpuspeed[key]["maxmhz"]:
            allclear=0
    return(allclear)

def screen_print(cpuspeed, cycle_counter, **kwargs):
    """
    kwargs:
    clear_screen = True
    Clear the screen instead of just going to home

    no_ansi = True
    Omit ansi characters for home, clear screen and colors

    no_runtime = True
    Do not print "ctrl+c" and seconds runtime

    """
    if "clear_screen" in kwargs:
        sys.stdout.write("%s%s" % (ansi.CLR,ansi.HOME))
    elif not "no_ansi" in kwargs:
        sys.stdout.write("%s" % (ansi.HOME,))
    print "A healthy and busy system should show MHz in increments of 100 (or XX01 at full speed) and hit full speed on all CPUs in 30-60 seconds.\n"
    for key, value in sorted(cpuspeed.items()):
        # "stuck" CPUs tend to stay at 1200MHz, so flag those as red
        # If the CPU hit the model's max, or 1 over max if it is static, flag it green
        # If the CPU is in between, mark it yellow
        print "CPU : %s, Model: %s, Max: %s" % (key,cpuspeed[key]["model"],cpuspeed[key]["maxmhz"])
        if not "no_ansi" in kwargs:
            if cpuspeed[key]["mhz"]==1200:
                status_color=ansi.RED
            elif not cpuspeed[key]["mhz"] >= cpuspeed[key]["maxmhz"]:
                status_color=ansi.YELLOW
            else:
                status_color=ansi.GREEN
        else:
            status_color=""
        print "Max observed speed: %s%s%s" % (status_color,cpuspeed[key]["mhz"],ansi.ENDC)
    if not "no_runtime" in kwargs:
        print "\nCtrl+C to exit, Runtime: %5.1f seconds" % (float(cycle_counter)/10,)
    return(allclear)

if ARGPARSE:
    parser = argparse.ArgumentParser()
    parser.add_argument("-r","--runtime", help="Maximum number of seconds to watch the CPUs before ending",
                        type=int)
    parser.add_argument("-s","--silent",
                        help="No output, return code of 0 if CPUs are clear, and non-zero if the CPUs do not hit max within the runtime specified. Default 30 seconds, use --runtime to specify the duration.",
                        action="store_true")
    parser.add_argument("-b","--batch", help="Run for a time, then output results. Default 30 seconds, use --runtime to specify the duration.",
                        action="store_true")
    parser.add_argument("--plaintext", help="ANSI control characters are omitted for colors and screen clear/home.",
                        action="store_true")
    args = parser.parse_args()
else:
    args = argsAlt()
    # dummy class in the event argparse is not available on the system

if not sys.stdout.isatty():
    # not interactively
    print "batch mode"
    args.batch = True

if (args.silent or args.batch) and not args.runtime:
    args.runtime = 30
    pass
if args.batch:
    args.plaintext = True
if args.batch:
    pass

cpuspeed = AutoVivification()
cycle_counter = 0
allclear = 0

if not (args.silent or args.plaintext):
    sys.stdout.write("%s%s" % (ansi.CLR,ansi.HOME))

try:
    while allclear == 0:
        infile = open('/proc/cpuinfo', 'r')
        cpuinfo = infile.readlines()
        cpu=""
        for line in cpuinfo:
            # cpu
            # model
            # mhz
            result = re.match('(processor|model name|cpu MHz)[\s:]+(\d*)(.*)', line.strip())
            # group 1 is category
            # group 2 is numbers (speed or processor number)
            # group 3 is anything following numbers (model name)

            if not result:
                continue
            if result.group(1)=="processor":
                # which CPU core are we talking about?
                cpu = int(result.group(2))
            elif result.group(1)=="model name" and not "model" in cpuspeed[cpu]:
                # we only set the model if it isn't already set, and we get the max speed at the same time too
                cpuspeed[cpu]["model"]=result.group(3)
                result = re.search('([\d\.]+)GHz', line.strip())
                cpuspeed[cpu]["maxmhz"]=int(float(re.search('([\d\.]+)GHz', line.strip()).group(1))*1000)
            elif result.group(1)=="cpu MHz":
                # you can't reference the variable that isn't set, so max alone doesn't work
                if not "mhz" in cpuspeed[cpu]:
                    cpuspeed[cpu]["mhz"]=int(result.group(2))
                else:
                    cpuspeed[cpu]["mhz"]=max(cpuspeed[cpu]["mhz"],int(result.group(2)))
        # assume all the CPUs are good (allclear), then test for lower states. Exit if they are all clear.
        allclear = all_clear(cpuspeed)
        if not (args.silent or args.batch):
            if args.plaintext:
                screen_print(cpuspeed, cycle_counter, no_ansi = True)
            else:
                screen_print(cpuspeed, cycle_counter)
        if args.runtime and (cycle_counter/10 >= args.runtime):
            allclear = 2
        infile.close()
        cycle_counter+=1
        sleep(.1)
except KeyboardInterrupt:
    #allclear = all_clear(cpuspeed)
    screen_print(cpuspeed, cycle_counter)
    sys.exit(128)

if args.plaintext:
    screen_print(cpuspeed, cycle_counter, no_ansi = True, no_runtime = True)
    pass

if allclear==1:
    if not args.silent:
        print "All CPUs cleared in %d seconds" % (cycle_counter/10)
    else:
        sys.exit(0)
elif allclear==2:
    plural=""
    if cycle_counter/10 > 1:
        plural="s"
    if not args.silent:
        print "CPUs did not clear during the test run, stopped after %d second%s" % ((cycle_counter/10), plural)
    sys.exit(1)<|endoftext|>


<commit_before>#!/usr/bin/python3

import argparse
import json
import os
import subprocess
import sys
import urllib.request
from operator import itemgetter

import requests
import requests_cache

__author__ = 'Marius Gedminas <marius@gedmin.as>'
__licence__ = 'MIT'
__url__ = 'https://github.com/mgedmin/cloneall'
__version__ = '1.2.dev0'

DEFAULT_ORGANIZATION = 'ZopeFoundation'

class Error(Exception):
    """An error that is not a bug in this script."""

def get_json_and_headers(url):
    """Perform HTTP GET for a URL, return deserialized JSON and headers.

    Returns a tuple (json_data, headers) where headers is something dict-like.
    """
    r = requests.get(url)
    if 400 <= r.status_code < 500:
        raise Error("Failed to fetch {}:\n{}".format(url, r.json()['message']))
    return r.json(), r.headers

def get_github_list(url, batch_size=100):
    """Perform (a series of) HTTP GETs for a URL, return deserialized JSON.

    Format of the JSON is documented at
    http://developer.github.com/v3/repos/#list-organization-repositories

    Supports batching (which Github indicates by the presence of a Link header,
    e.g. ::

        Link: <https://api.github.com/resource?page=2>; rel="next",
              <https://api.github.com/resource?page=5>; rel="last"

    """
    # API documented at http://developer.github.com/v3/#pagination
    res, headers = get_json_and_headers('{}?per_page={}'.format(
                                                url, batch_size))
    page = 1
    while 'rel="next"' in headers.get('Link', ''):
        page += 1
        more, headers = get_json_and_headers('{}?page={}&per_page={}'.format(
                                                    url, page, batch_size))
        res += more
    return res

class Progress(object):
    stream = sys.stdout
    last_message = ''
    format = '[{bar}] {cur}/{total}'
    bar_width = 20
    last_item = ''
    t_cursor_up = '\033[A'
    t_reset = '\033[m'
    t_green = '\033[32m'

    def status(self, message):
        if self.last_message:
            self.clear()
        self.stream.write('\r')
        self.stream.write(message)
        self.stream.write('\r')
        self.stream.flush()
        self.last_message = message

    def clear(self):
        self.stream.write('\r{}\r'.format(' ' * len(self.last_message.rstrip())))
        self.stream.flush()
        self.last_message = ''

    def message(self, cur, total):
        return self.format.format(cur=cur, total=total,
                                  bar=self.bar(cur, total))

    def scale(self, range, cur, total):
        return range * cur // max(total, 1)

    def bar(self, cur, total):
        n = self.scale(self.bar_width, cur, total)
        return ('=' * n).ljust(self.bar_width)

    def __call__(self, cur, total):
        self.status(self.message(cur, total))

    def item(self, msg):
        print(msg, file=self.stream)
        self.last_item = msg

    def update(self, msg, color=t_green):
        self.last_item += msg
        print(''.join([self.t_cursor_up, color, self.last_item, self.t_reset]),
              file=self.stream)

def main():
    parser = argparse.ArgumentParser(
        description="Clone/update all organization repositories from GitHub")
    parser.add_argument('--version', action='version',
                        version="%(prog)s version " + __version__)
    parser.add_argument('-n', '--dry-run', action='store_true',
                        help="don't pull/clone, just print what would be done")
    parser.add_argument('-v', '--verbose', action='count',
                        help="perform additional checks")
    parser.add_argument('--start-from', metavar='REPO',
                        help='skip all repositories that come before REPO alphabetically')
    parser.add_argument('--organization', default=DEFAULT_ORGANIZATION,
                        help='specify the GitHub organization (default: %s)' % DEFAULT_ORGANIZATION)
    parser.add_argument('--http-cache', default='.httpcache', metavar='DBNAME',
                        # .sqlite will be appended automatically
                        help='cache HTTP requests on disk in an sqlite database (default: .httpcache)')
    parser.add_argument('--no-http-cache', action='store_false', dest='http_cache',
                        help='disable HTTP disk caching')
    args = parser.parse_args()
    if args.http_cache:
        requests_cache.install_cache(args.http_cache,
                                     backend='sqlite',
                                     expire_after=300)

    progress = Progress()
    progress.status('Fetching list of {} repositories from GitHub...'.format(args.organization))
    list_url = 'https://api.github.com/orgs/{}/repos'.format(args.organization)
    repos = sorted(get_github_list(list_url), key=itemgetter('name'))
    progress.clear()
    n_fetched = n_updated = n_new = n_dirty = 0
    for n, repo in enumerate(repos, 1):
        if args.start_from and repo['name'] < args.start_from:
            continue
        progress.item("+ {name}".format(**repo))
        progress(n, len(repos))
        n_fetched += 1
        dir = repo['name']
        if os.path.exists(dir):
            if not args.dry_run:
                old_sha = subprocess.check_output(['git', 'describe', '--always', '--dirty'], cwd=dir)
                subprocess.call(['git', 'pull', '-q', '--ff-only'], cwd=dir)
                new_sha = subprocess.check_output(['git', 'describe', '--always', '--dirty'], cwd=dir)
                if old_sha != new_sha:
                    progress.update(' (updated)')
                    n_updated += 1
            # commands borrowed from /usr/lib/git-core/git-sh-prompt
            dirty = 0
            if subprocess.call(['git', 'diff', '--no-ext-diff', '--quiet', '--exit-code'], cwd=dir) != 0:
                progress.update(' (local changes)')
                dirty = 1
            if subprocess.call(['git', 'diff-index', '--cached', '--quiet', 'HEAD', '--'], cwd=dir) != 0:
                progress.update(' (staged changes)')
                dirty = 1
            if subprocess.check_output(['git', 'rev-list', '@{u}..'], cwd=dir) != b'':
                progress.update(' (local commits)')
                dirty = 1
            if subprocess.check_output(['git', 'symbolic-ref', 'HEAD'], cwd=dir) != b'refs/heads/master\n':
                progress.update(' (not on master)')
                dirty = 1
            if args.verbose:
                remote_url = subprocess.check_output(['git', 'ls-remote', '--get-url'], cwd=dir)
                remote_url = remote_url.decode('UTF-8', 'replace').strip()
                if remote_url != repo['ssh_url'] and remote_url + '.git' != repo['ssh_url']:
                    progress.update(' (wrong remote url)')
                    if args.verbose >= 2:
                        progress.status('')
                        print('   ', remote_url)
                    dirty = 1
                unknown_files = subprocess.check_output(['git', 'ls-files', '--others', '--exclude-standard', '--', ':/*'], cwd=dir)
                if unknown_files:
                    progress.update(' (unknown files)')
                    if args.verbose >= 2:
                        progress.status('')
                        files = unknown_files.decode('UTF-8', 'replace').splitlines()
                        for n, fn in enumerate(files):
                            if args.verbose < 3 and n == 10:
                                print('    (and %d more)' % (len(files) - n))
                                break
                            print('   ', fn)
                    dirty = 1
            n_dirty += dirty
        else:
            if not args.dry_run:
                # use repo['git_url'] for anonymous checkouts
                subprocess.call(['git', 'clone', '-q', repo['ssh_url']])
            progress.update(' (new)')
            n_new += 1
        progress.clear()
    print("{n_fetched} repositories: {n_updated} updated, {n_new} new, {n_dirty} dirty.".format(
          n_fetched=n_fetched, n_updated=n_updated, n_new=n_new, n_dirty=n_dirty))

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        pass
<commit_msg>Declare this to be 1.2<commit_after>#!/usr/bin/python3

import argparse
import json
import os
import subprocess
import sys
import urllib.request
from operator import itemgetter

import requests
import requests_cache

__author__ = 'Marius Gedminas <marius@gedmin.as>'
__licence__ = 'MIT'
__url__ = 'https://github.com/mgedmin/cloneall'
__version__ = '1.2'

DEFAULT_ORGANIZATION = 'ZopeFoundation'

class Error(Exception):
    """An error that is not a bug in this script."""

def get_json_and_headers(url):
    """Perform HTTP GET for a URL, return deserialized JSON and headers.

    Returns a tuple (json_data, headers) where headers is something dict-like.
    """
    r = requests.get(url)
    if 400 <= r.status_code < 500:
        raise Error("Failed to fetch {}:\n{}".format(url, r.json()['message']))
    return r.json(), r.headers

def get_github_list(url, batch_size=100):
    """Perform (a series of) HTTP GETs for a URL, return deserialized JSON.

    Format of the JSON is documented at
    http://developer.github.com/v3/repos/#list-organization-repositories

    Supports batching (which Github indicates by the presence of a Link header,
    e.g. ::

        Link: <https://api.github.com/resource?page=2>; rel="next",
              <https://api.github.com/resource?page=5>; rel="last"

    """
    # API documented at http://developer.github.com/v3/#pagination
    res, headers = get_json_and_headers('{}?per_page={}'.format(
                                                url, batch_size))
    page = 1
    while 'rel="next"' in headers.get('Link', ''):
        page += 1
        more, headers = get_json_and_headers('{}?page={}&per_page={}'.format(
                                                    url, page, batch_size))
        res += more
    return res

class Progress(object):
    stream = sys.stdout
    last_message = ''
    format = '[{bar}] {cur}/{total}'
    bar_width = 20
    last_item = ''
    t_cursor_up = '\033[A'
    t_reset = '\033[m'
    t_green = '\033[32m'

    def status(self, message):
        if self.last_message:
            self.clear()
        self.stream.write('\r')
        self.stream.write(message)
        self.stream.write('\r')
        self.stream.flush()
        self.last_message = message

    def clear(self):
        self.stream.write('\r{}\r'.format(' ' * len(self.last_message.rstrip())))
        self.stream.flush()
        self.last_message = ''

    def message(self, cur, total):
        return self.format.format(cur=cur, total=total,
                                  bar=self.bar(cur, total))

    def scale(self, range, cur, total):
        return range * cur // max(total, 1)

    def bar(self, cur, total):
        n = self.scale(self.bar_width, cur, total)
        return ('=' * n).ljust(self.bar_width)

    def __call__(self, cur, total):
        self.status(self.message(cur, total))

    def item(self, msg):
        print(msg, file=self.stream)
        self.last_item = msg

    def update(self, msg, color=t_green):
        self.last_item += msg
        print(''.join([self.t_cursor_up, color, self.last_item, self.t_reset]),
              file=self.stream)

def main():
    parser = argparse.ArgumentParser(
        description="Clone/update all organization repositories from GitHub")
    parser.add_argument('--version', action='version',
                        version="%(prog)s version " + __version__)
    parser.add_argument('-n', '--dry-run', action='store_true',
                        help="don't pull/clone, just print what would be done")
    parser.add_argument('-v', '--verbose', action='count',
                        help="perform additional checks")
    parser.add_argument('--start-from', metavar='REPO',
                        help='skip all repositories that come before REPO alphabetically')
    parser.add_argument('--organization', default=DEFAULT_ORGANIZATION,
                        help='specify the GitHub organization (default: %s)' % DEFAULT_ORGANIZATION)
    parser.add_argument('--http-cache', default='.httpcache', metavar='DBNAME',
                        # .sqlite will be appended automatically
                        help='cache HTTP requests on disk in an sqlite database (default: .httpcache)')
    parser.add_argument('--no-http-cache', action='store_false', dest='http_cache',
                        help='disable HTTP disk caching')
    args = parser.parse_args()
    if args.http_cache:
        requests_cache.install_cache(args.http_cache,
                                     backend='sqlite',
                                     expire_after=300)

    progress = Progress()
    progress.status('Fetching list of {} repositories from GitHub...'.format(args.organization))
    list_url = 'https://api.github.com/orgs/{}/repos'.format(args.organization)
    repos = sorted(get_github_list(list_url), key=itemgetter('name'))
    progress.clear()
    n_fetched = n_updated = n_new = n_dirty = 0
    for n, repo in enumerate(repos, 1):
        if args.start_from and repo['name'] < args.start_from:
            continue
        progress.item("+ {name}".format(**repo))
        progress(n, len(repos))
        n_fetched += 1
        dir = repo['name']
        if os.path.exists(dir):
            if not args.dry_run:
                old_sha = subprocess.check_output(['git', 'describe', '--always', '--dirty'], cwd=dir)
                subprocess.call(['git', 'pull', '-q', '--ff-only'], cwd=dir)
                new_sha = subprocess.check_output(['git', 'describe', '--always', '--dirty'], cwd=dir)
                if old_sha != new_sha:
                    progress.update(' (updated)')
                    n_updated += 1
            # commands borrowed from /usr/lib/git-core/git-sh-prompt
            dirty = 0
            if subprocess.call(['git', 'diff', '--no-ext-diff', '--quiet', '--exit-code'], cwd=dir) != 0:
                progress.update(' (local changes)')
                dirty = 1
            if subprocess.call(['git', 'diff-index', '--cached', '--quiet', 'HEAD', '--'], cwd=dir) != 0:
                progress.update(' (staged changes)')
                dirty = 1
            if subprocess.check_output(['git', 'rev-list', '@{u}..'], cwd=dir) != b'':
                progress.update(' (local commits)')
                dirty = 1
            if subprocess.check_output(['git', 'symbolic-ref', 'HEAD'], cwd=dir) != b'refs/heads/master\n':
                progress.update(' (not on master)')
                dirty = 1
            if args.verbose:
                remote_url = subprocess.check_output(['git', 'ls-remote', '--get-url'], cwd=dir)
                remote_url = remote_url.decode('UTF-8', 'replace').strip()
                if remote_url != repo['ssh_url'] and remote_url + '.git' != repo['ssh_url']:
                    progress.update(' (wrong remote url)')
                    if args.verbose >= 2:
                        progress.status('')
                        print('   ', remote_url)
                    dirty = 1
                unknown_files = subprocess.check_output(['git', 'ls-files', '--others', '--exclude-standard', '--', ':/*'], cwd=dir)
                if unknown_files:
                    progress.update(' (unknown files)')
                    if args.verbose >= 2:
                        progress.status('')
                        files = unknown_files.decode('UTF-8', 'replace').splitlines()
                        for n, fn in enumerate(files):
                            if args.verbose < 3 and n == 10:
                                print('    (and %d more)' % (len(files) - n))
                                break
                            print('   ', fn)
                    dirty = 1
            n_dirty += dirty
        else:
            if not args.dry_run:
                # use repo['git_url'] for anonymous checkouts
                subprocess.call(['git', 'clone', '-q', repo['ssh_url']])
            progress.update(' (new)')
            n_new += 1
        progress.clear()
    print("{n_fetched} repositories: {n_updated} updated, {n_new} new, {n_dirty} dirty.".format(
          n_fetched=n_fetched, n_updated=n_updated, n_new=n_new, n_dirty=n_dirty))

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        pass
<|endoftext|>


<commit_before>"""Models for closuretree application."""

# We like magic.
# pylint: disable=W0142

# We have lots of dynamically generated things, hard for pylint to solve.
# pylint: disable=E1101

# It may not be our class, but we made the attribute on it
# pylint: disable=W0212

# Public methods are useful!
# pylint: disable=R0904

from django.db import models
from django.db.models.base import ModelBase
import sys

def _closure_model_unicode(self):
    """__unicode__ implementation for the dynamically created
        <Model>Closure model.
    """
    return "Closure from %s to %s" % (self.parent, self.child)

def create_closure_model(cls):
    """Creates a <Model>Closure model in the same module as the model."""
    model = type('%sClosure' % cls.__name__, (models.Model,), {
        'parent': models.ForeignKey(
            cls.__name__,
            related_name=cls.closure_parentref()
        ),
        'child': models.ForeignKey(
            cls.__name__,
            related_name=cls.closure_childref()
        ),
        'depth': models.IntegerField(),
        '__module__':   cls.__module__,
        '__unicode__': _closure_model_unicode,
        'Meta': type('Meta', (object,), {
            'unique_together':  (("parent", "child"),)
        }),
    })
    setattr(cls, "_closure_model", model)
    return model

class ClosureModelBase(ModelBase):
    """Metaclass for Models inheriting from ClosureModel,
        to ensure the <Model>Closure model is created.
    """
    #This is a metaclass. MAGIC!
    def __init__(cls, name, bases, dct):
        """Create the closure model in addition
            to doing all the normal django stuff.
        """
        super(ClosureModelBase, cls).__init__(name, bases, dct)
        if not cls._meta.get_parent_list() and cls.__module__ != __name__:
            setattr(
                sys.modules[cls.__module__],
                '%sClosure' % cls.__name__,
                create_closure_model(cls)
            )

class ClosureModel(models.Model):
    """Provides methods to assist in a tree based structure."""
    __metaclass__ = ClosureModelBase

    class Meta:
        """We make this an abstract class, it needs to be inherited from."""
        # pylint: disable=W0232
        # pylint: disable=R0903
        abstract = True

    def __setattr__(self, name, value):
        if (
            name.startswith(self._closure_sentinel_attr) and
            hasattr(self, name) and
            not self._closure_change_check()
        ):
            # Already set once, and not already stored the old
            # value, need to take a copy before it changes
            self._closure_change_init()
        super(ClosureModel, self).__setattr__(name, value)

    @classmethod
    def _toplevel(cls):
        """Find the top level of the chain we're in.

            For example, if we have:
            C inheriting from B inheriting from A inheriting from ClosureModel
            C._toplevel() will return A.
        """
        superclasses = (
            set(ClosureModel.__subclasses__()) &
            cls._meta.get_parent_list()
        )
        return next(iter(superclasses)) if superclasses else cls

    @classmethod
    def rebuildtable(cls):
        """Regenerate the entire closuretree."""
        cls._closure_model.objects.all().delete()
        cls._closure_model.objects.bulk_create([cls._closure_model(
            parent_id=x['pk'],
            child_id=x['pk'],
            depth=0
        ) for x in cls.objects.values("pk")])
        for node in cls.objects.all():
            node._closure_createlink()

    @classmethod
    def closure_parentref(cls):
        """How to refer to parents in the closure tree"""
        return "%sclosure_children" % cls._toplevel().__name__.lower()

    # Backwards compatibility:
    _closure_parentref = closure_parentref

    @classmethod
    def closure_childref(cls):
        """How to refer to children in the closure tree"""
        return "%sclosure_parents" % cls._toplevel().__name__.lower()

    # Backwards compatibility:
    _closure_childref = closure_childref

    @property
    def _closure_sentinel_attr(self):
        """The attribute we need to watch to tell if the
            parent/child relationships have changed
        """
        if hasattr(self.ClosureMeta,"sentinel_attr"):
            return self.ClosureMeta.sentinel_attr
        else:
            return self.ClosureMeta.parent_attr

    @property
    def _closure_parent_pk(self):
        """What our parent pk is in the closure tree."""
        if hasattr(self, "%s_id" % self.ClosureMeta.parent_attr):
            return getattr(self, "%s_id" % self.ClosureMeta.parent_attr)
        else:
            parent = getattr(self, self.ClosureMeta.parent_attr)
            return parent.id if parent else None

    def _closure_deletelink(self, oldparentpk):
        """Remove incorrect links from the closure tree."""
        self._closure_model.objects.filter(
            **{
                "parent__%s__child" % self._closure_parentref(): oldparentpk,
                "child__%s__parent" % self._closure_childref(): self.pk
            }
        ).delete()

    def _closure_createlink(self):
        """Create a link in the closure tree."""
        linkparents = self._closure_model.objects.filter(
            child__id=self._closure_parent_pk
        ).values("parent", "depth")
        linkchildren = self._closure_model.objects.filter(
            parent__id=self.pk
        ).values("child", "depth")
        newlinks = [self._closure_model(
            parent_id=p['parent'],
            child_id=c['child'],
            depth=p['depth']+c['depth']+1
        ) for p in linkparents for c in linkchildren]
        self._closure_model.objects.bulk_create(newlinks)

    def get_ancestors(self, include_self=False, depth=None):
        """Return all the ancestors of this object."""
        if self.is_root_node():
            if not include_self:
                return self._toplevel().objects.none()
            else:
                # Filter on pk for efficiency.
                return self._toplevel().objects.filter(pk=self.pk)

        params = {"%s__child" % self._closure_parentref():self.pk}
        if depth is not None:
            params["%s__depth__lte" % self._closure_parentref()] = depth
        ancestors = self._toplevel().objects.filter(**params)
        if not include_self:
            ancestors = ancestors.exclude(pk=self.pk)
        return ancestors.order_by("%s__depth" % self._closure_parentref())

    def get_descendants(self, include_self=False, depth=None):
        """Return all the descendants of this object."""
        params = {"%s__parent" % self._closure_childref():self.pk}
        if depth is not None:
            params["%s__depth__lte" % self._closure_childref()] = depth
        descendants = self._toplevel().objects.filter(**params)
        if not include_self:
            descendants = descendants.exclude(pk=self.pk)
        return descendants.order_by("%s__depth" % self._closure_childref())

    def prepopulate(self, queryset):
        """Perpopulate a descendants query's children efficiently.
            Call like: blah.prepopulate(blah.get_descendants().select_related(stuff))
        """
        objs = list(queryset)
        hashobjs = dict([(x.pk, x) for x in objs] + [(self.pk, self)])
        for descendant in hashobjs.values():
            descendant._cached_children = []
        for descendant in objs:
            if descendant._closure_parent_pk in hashobjs:
                parent = hashobjs[descendant._closure_parent_pk]
                parent._cached_children.append(descendant)

    def get_children(self):
        """Return all the children of this object."""
        if hasattr(self, '_cached_children'):
            children = self._toplevel().objects.filter(
                pk__in=[n.pk for n in self._cached_children]
            )
            children._result_cache = self._cached_children
            return children
        else:
            return self.get_descendants(include_self=False, depth=1)

    def get_root(self):
        """Return the furthest ancestor of this node."""
        if self.is_root_node():
            return self

        return self.get_ancestors().order_by(
            "-%s__depth" % self._closure_parentref()
        )[0]

    def is_child_node(self):
        """Is this node a child, i.e. has a parent?"""
        return not self.is_root_node()

    def is_root_node(self):
        """Is this node a root, i.e. has no parent?"""
        return self._closure_parent_pk is None

    def is_descendant_of(self, other, include_self=False):
        """Is this node a descendant of `other`?"""
        if other.pk == self.pk:
            return include_self

        return self._closure_model.objects.filter(
            parent=other,
            child=self
        ).exclude(pk=self.pk).exists()

    def is_ancestor_of(self, other, include_self=False):
        """Is this node an ancestor of `other`?"""
        return other.is_descendant_of(self, include_self=include_self)

    def _closure_change_init(self):
        """Part of the change detection. Setting up"""
        # More magic. We're setting this inside setattr...
        # pylint: disable=W0201
        self._closure_old_parent_pk = self._closure_parent_pk

    def _closure_change_check(self):
        """Part of the change detection. Have we changed since we began?"""
        return hasattr(self,"_closure_old_parent_pk")

    def _closure_change_oldparent(self):
        """Part of the change detection. What we used to be"""
        return self._closure_old_parent_pk

    def save(self, *args, **kwargs):
        create = not self.id
        val = super(ClosureModel, self).save(*args, **kwargs)
        if create:
            closure_instance = self._closure_model(
                parent=self,
                child=self,
                depth=0
            )
            closure_instance.save()
        if self._closure_change_check():
            #Changed parents.
            if self._closure_change_oldparent():
                self._closure_deletelink(self._closure_change_oldparent())
            self._closure_createlink()
            delattr(self, "_closure_old_parent_pk")
        elif create:
            # We still need to create links when we're first made
            self._closure_createlink()

        return val

    def delete(self, *args, **kwargs):
        self._closure_deletelink(self._closure_parent_pk)
        super(ClosureModel, self).delete(*args, **kwargs)
<commit_msg>That if should never be false, make it an assert<commit_after>"""Models for closuretree application."""

# We like magic.
# pylint: disable=W0142

# We have lots of dynamically generated things, hard for pylint to solve.
# pylint: disable=E1101

# It may not be our class, but we made the attribute on it
# pylint: disable=W0212

# Public methods are useful!
# pylint: disable=R0904

from django.db import models
from django.db.models.base import ModelBase
import sys

def _closure_model_unicode(self):
    """__unicode__ implementation for the dynamically created
        <Model>Closure model.
    """
    return "Closure from %s to %s" % (self.parent, self.child)

def create_closure_model(cls):
    """Creates a <Model>Closure model in the same module as the model."""
    model = type('%sClosure' % cls.__name__, (models.Model,), {
        'parent': models.ForeignKey(
            cls.__name__,
            related_name=cls.closure_parentref()
        ),
        'child': models.ForeignKey(
            cls.__name__,
            related_name=cls.closure_childref()
        ),
        'depth': models.IntegerField(),
        '__module__':   cls.__module__,
        '__unicode__': _closure_model_unicode,
        'Meta': type('Meta', (object,), {
            'unique_together':  (("parent", "child"),)
        }),
    })
    setattr(cls, "_closure_model", model)
    return model

class ClosureModelBase(ModelBase):
    """Metaclass for Models inheriting from ClosureModel,
        to ensure the <Model>Closure model is created.
    """
    #This is a metaclass. MAGIC!
    def __init__(cls, name, bases, dct):
        """Create the closure model in addition
            to doing all the normal django stuff.
        """
        super(ClosureModelBase, cls).__init__(name, bases, dct)
        if not cls._meta.get_parent_list() and cls.__module__ != __name__:
            setattr(
                sys.modules[cls.__module__],
                '%sClosure' % cls.__name__,
                create_closure_model(cls)
            )

class ClosureModel(models.Model):
    """Provides methods to assist in a tree based structure."""
    __metaclass__ = ClosureModelBase

    class Meta:
        """We make this an abstract class, it needs to be inherited from."""
        # pylint: disable=W0232
        # pylint: disable=R0903
        abstract = True

    def __setattr__(self, name, value):
        if (
            name.startswith(self._closure_sentinel_attr) and
            hasattr(self, name) and
            not self._closure_change_check()
        ):
            # Already set once, and not already stored the old
            # value, need to take a copy before it changes
            self._closure_change_init()
        super(ClosureModel, self).__setattr__(name, value)

    @classmethod
    def _toplevel(cls):
        """Find the top level of the chain we're in.

            For example, if we have:
            C inheriting from B inheriting from A inheriting from ClosureModel
            C._toplevel() will return A.
        """
        superclasses = (
            set(ClosureModel.__subclasses__()) &
            cls._meta.get_parent_list()
        )
        return next(iter(superclasses)) if superclasses else cls

    @classmethod
    def rebuildtable(cls):
        """Regenerate the entire closuretree."""
        cls._closure_model.objects.all().delete()
        cls._closure_model.objects.bulk_create([cls._closure_model(
            parent_id=x['pk'],
            child_id=x['pk'],
            depth=0
        ) for x in cls.objects.values("pk")])
        for node in cls.objects.all():
            node._closure_createlink()

    @classmethod
    def closure_parentref(cls):
        """How to refer to parents in the closure tree"""
        return "%sclosure_children" % cls._toplevel().__name__.lower()

    # Backwards compatibility:
    _closure_parentref = closure_parentref

    @classmethod
    def closure_childref(cls):
        """How to refer to children in the closure tree"""
        return "%sclosure_parents" % cls._toplevel().__name__.lower()

    # Backwards compatibility:
    _closure_childref = closure_childref

    @property
    def _closure_sentinel_attr(self):
        """The attribute we need to watch to tell if the
            parent/child relationships have changed
        """
        if hasattr(self.ClosureMeta,"sentinel_attr"):
            return self.ClosureMeta.sentinel_attr
        else:
            return self.ClosureMeta.parent_attr

    @property
    def _closure_parent_pk(self):
        """What our parent pk is in the closure tree."""
        if hasattr(self, "%s_id" % self.ClosureMeta.parent_attr):
            return getattr(self, "%s_id" % self.ClosureMeta.parent_attr)
        else:
            parent = getattr(self, self.ClosureMeta.parent_attr)
            return parent.id if parent else None

    def _closure_deletelink(self, oldparentpk):
        """Remove incorrect links from the closure tree."""
        self._closure_model.objects.filter(
            **{
                "parent__%s__child" % self._closure_parentref(): oldparentpk,
                "child__%s__parent" % self._closure_childref(): self.pk
            }
        ).delete()

    def _closure_createlink(self):
        """Create a link in the closure tree."""
        linkparents = self._closure_model.objects.filter(
            child__id=self._closure_parent_pk
        ).values("parent", "depth")
        linkchildren = self._closure_model.objects.filter(
            parent__id=self.pk
        ).values("child", "depth")
        newlinks = [self._closure_model(
            parent_id=p['parent'],
            child_id=c['child'],
            depth=p['depth']+c['depth']+1
        ) for p in linkparents for c in linkchildren]
        self._closure_model.objects.bulk_create(newlinks)

    def get_ancestors(self, include_self=False, depth=None):
        """Return all the ancestors of this object."""
        if self.is_root_node():
            if not include_self:
                return self._toplevel().objects.none()
            else:
                # Filter on pk for efficiency.
                return self._toplevel().objects.filter(pk=self.pk)

        params = {"%s__child" % self._closure_parentref():self.pk}
        if depth is not None:
            params["%s__depth__lte" % self._closure_parentref()] = depth
        ancestors = self._toplevel().objects.filter(**params)
        if not include_self:
            ancestors = ancestors.exclude(pk=self.pk)
        return ancestors.order_by("%s__depth" % self._closure_parentref())

    def get_descendants(self, include_self=False, depth=None):
        """Return all the descendants of this object."""
        params = {"%s__parent" % self._closure_childref():self.pk}
        if depth is not None:
            params["%s__depth__lte" % self._closure_childref()] = depth
        descendants = self._toplevel().objects.filter(**params)
        if not include_self:
            descendants = descendants.exclude(pk=self.pk)
        return descendants.order_by("%s__depth" % self._closure_childref())

    def prepopulate(self, queryset):
        """Perpopulate a descendants query's children efficiently.
            Call like: blah.prepopulate(blah.get_descendants().select_related(stuff))
        """
        objs = list(queryset)
        hashobjs = dict([(x.pk, x) for x in objs] + [(self.pk, self)])
        for descendant in hashobjs.values():
            descendant._cached_children = []
        for descendant in objs:
            assert descendant._closure_parent_pk in hashobjs
            parent = hashobjs[descendant._closure_parent_pk]
            parent._cached_children.append(descendant)

    def get_children(self):
        """Return all the children of this object."""
        if hasattr(self, '_cached_children'):
            children = self._toplevel().objects.filter(
                pk__in=[n.pk for n in self._cached_children]
            )
            children._result_cache = self._cached_children
            return children
        else:
            return self.get_descendants(include_self=False, depth=1)

    def get_root(self):
        """Return the furthest ancestor of this node."""
        if self.is_root_node():
            return self

        return self.get_ancestors().order_by(
            "-%s__depth" % self._closure_parentref()
        )[0]

    def is_child_node(self):
        """Is this node a child, i.e. has a parent?"""
        return not self.is_root_node()

    def is_root_node(self):
        """Is this node a root, i.e. has no parent?"""
        return self._closure_parent_pk is None

    def is_descendant_of(self, other, include_self=False):
        """Is this node a descendant of `other`?"""
        if other.pk == self.pk:
            return include_self

        return self._closure_model.objects.filter(
            parent=other,
            child=self
        ).exclude(pk=self.pk).exists()

    def is_ancestor_of(self, other, include_self=False):
        """Is this node an ancestor of `other`?"""
        return other.is_descendant_of(self, include_self=include_self)

    def _closure_change_init(self):
        """Part of the change detection. Setting up"""
        # More magic. We're setting this inside setattr...
        # pylint: disable=W0201
        self._closure_old_parent_pk = self._closure_parent_pk

    def _closure_change_check(self):
        """Part of the change detection. Have we changed since we began?"""
        return hasattr(self,"_closure_old_parent_pk")

    def _closure_change_oldparent(self):
        """Part of the change detection. What we used to be"""
        return self._closure_old_parent_pk

    def save(self, *args, **kwargs):
        create = not self.id
        val = super(ClosureModel, self).save(*args, **kwargs)
        if create:
            closure_instance = self._closure_model(
                parent=self,
                child=self,
                depth=0
            )
            closure_instance.save()
        if self._closure_change_check():
            #Changed parents.
            if self._closure_change_oldparent():
                self._closure_deletelink(self._closure_change_oldparent())
            self._closure_createlink()
            delattr(self, "_closure_old_parent_pk")
        elif create:
            # We still need to create links when we're first made
            self._closure_createlink()

        return val

    def delete(self, *args, **kwargs):
        self._closure_deletelink(self._closure_parent_pk)
        super(ClosureModel, self).delete(*args, **kwargs)
<|endoftext|>


<commit_before>
from assertions import *
from raster import *
from rectilinear import *
from structured import *
from unstructured import *
from field import *
from igrid import *
try:
    from esmp import *
except ImportError:
    from warnings import warn
    warn ('ESMF mapper disabled', RuntimeWarning)
try:
    from map import *
except (ImportError, OSError):
    from warnings import warn
    warn ('CSDMS mapper disabled', RuntimeWarning)
<commit_msg>* Changed to import select functions and classes rather than everything.<commit_after>from cmt.grids.raster import UniformRectilinear
from cmt.grids.rectilinear import RectilinearPoints, Rectilinear
from cmt.grids.structured import Structured
from cmt.grids.unstructured import Unstructured

from cmt.grids.assertions import (is_uniform_rectilinear, is_rectilinear,
                                  is_structured, is_unstructured)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Wed Deb 14 14:21:41 2018
Script to query TNS with different parameters:
either for transients discovered between two different dates,
or a cone search radius around a given RA, DEC

@author: nadiablago
"""

try:
    # For Python 3.0 and later
    from urllib.request import urlopen
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen

import sys, os
#reload(sys)
#sys.setdefaultencoding('utf8')

import numpy as np
from astropy.table import Table
from astropy.coordinates import SkyCoord
from astropy.table import Column
import astropy.units as u
import sys
if sys.version_info[0] < 3:
    from StringIO import StringIO
else:
    from io import StringIO

def get_tns_date2date(date1, date2):
    '''
    Queries the TNS and obtains the targets reported between two dates.
    It parses the coordinates and transporms them into decimals.
    It writes a csv table with RA, DEC in degrees, which is directly ingestable into a postresql file.

    date1: in the format of: YYYY-MM-DD
    date2: in the format of: YYYY-MM-DD

    '''

    url = "https://wis-tns.weizmann.ac.il/search?&date_start%5Bdate%5D={0}&date_end%5Bdate%5D={1}&format=csv".format(date1, date2)
    cont_url = urlopen(url)
    cont = cont_url.read()

    print cont

    t = Table.read(StringIO(cont), format='csv')
    coords = np.array([t["RA"], t["DEC"]]).T
    c = SkyCoord(coords, frame='icrs', unit=(u.hourangle, u.deg))

    radeg = Column(c.ra, name='RA')
    decdeg = Column(c.dec, name='DEC')

    t.remove_column("RA")
    t.remove_column("DEC")
    t.add_column(radeg, index=1)
    t.add_column(decdeg, index=2)

    t.write("tns_query_%s_%s.csv"%(date1, date2), format="csv")

    return t

def get_tns_ra_dec(ra, dec, rad=15):
    '''
    Queries the TNS and obtains the targets reported for the specified RA, DEC position.
    Provided that ASASSN targets are there, a 7 arcsec position error is expected.
    By default we will use 10 arcsec.

    ra: float
        position in degrees
    dec: float
        position in degrees
    rad: float, optional
        Search radius in arcseconds.

    '''

    url = "https://wis-tns.weizmann.ac.il/search?&name=&ra={0}&decl={1}&radius={2}&coords_unit=arcsec&format=csv".format(ra, dec, rad)
    cont_url = urlopen(url)
    cont = cont_url.read()

    t = Table.read(StringIO(cont), format='ascii.csv')

    if len(t) > 0:
        coords = np.array([t["RA"], t["DEC"]]).T
        c = SkyCoord(coords, frame='icrs', unit=(u.hourangle, u.deg))
        basecoord = SkyCoord(ra, dec,  frame='icrs', unit=(u.deg, u.deg))

        #In case there are several objects in the match radius, we select the closest one
        dist = c.separation(basecoord)

        closest = t[np.argmin(dist)]
    else:
        closest = None

    return closest<commit_msg>The TNS query was limited to 50 rows. Now the limit is 1000.<commit_after># -*- coding: utf-8 -*-
"""
Created on Wed Deb 14 14:21:41 2018
Script to query TNS with different parameters:
either for transients discovered between two different dates,
or a cone search radius around a given RA, DEC

@author: nadiablago
"""

try:
    # For Python 3.0 and later
    from urllib.request import urlopen
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen

import sys, os
#reload(sys)
#sys.setdefaultencoding('utf8')

import numpy as np
from astropy.table import Table
from astropy.coordinates import SkyCoord
from astropy.table import Column
import astropy.units as u
import sys
if sys.version_info[0] < 3:
    from StringIO import StringIO
else:
    from io import StringIO

def get_tns_date2date(date1, date2):
    '''
    Queries the TNS and obtains the targets reported between two dates.
    It parses the coordinates and transporms them into decimals.
    It writes a csv table with RA, DEC in degrees, which is directly ingestable into a postresql file.

    date1: in the format of: YYYY-MM-DD
    date2: in the format of: YYYY-MM-DD

    '''

    url = "https://wis-tns.weizmann.ac.il/search?&date_start%5Bdate%5D={0}&date_end%5Bdate%5D={1}&format=csv&num_page=1000".format(date1, date2)
    cont_url = urlopen(url)
    cont = cont_url.read()

    print cont

    t = Table.read(StringIO(cont), format='csv')
    coords = np.array([t["RA"], t["DEC"]]).T
    c = SkyCoord(coords, frame='icrs', unit=(u.hourangle, u.deg))

    radeg = Column(c.ra, name='RA')
    decdeg = Column(c.dec, name='DEC')

    t.remove_column("RA")
    t.remove_column("DEC")
    t.add_column(radeg, index=1)
    t.add_column(decdeg, index=2)

    t.write("tns_query_%s_%s.csv"%(date1, date2), format="csv")

    return t

def get_tns_ra_dec(ra, dec, rad=15):
    '''
    Queries the TNS and obtains the targets reported for the specified RA, DEC position.
    Provided that ASASSN targets are there, a 7 arcsec position error is expected.
    By default we will use 10 arcsec.

    ra: float
        position in degrees
    dec: float
        position in degrees
    rad: float, optional
        Search radius in arcseconds.

    '''

    url = "https://wis-tns.weizmann.ac.il/search?&name=&ra={0}&decl={1}&radius={2}&coords_unit=arcsec&format=csv".format(ra, dec, rad)
    cont_url = urlopen(url)
    cont = cont_url.read()

    t = Table.read(StringIO(cont), format='ascii.csv')

    if len(t) > 0:
        coords = np.array([t["RA"], t["DEC"]]).T
        c = SkyCoord(coords, frame='icrs', unit=(u.hourangle, u.deg))
        basecoord = SkyCoord(ra, dec,  frame='icrs', unit=(u.deg, u.deg))

        #In case there are several objects in the match radius, we select the closest one
        dist = c.separation(basecoord)

        closest = t[np.argmin(dist)]
    else:
        closest = None

    return closest<|endoftext|>


<commit_before>#
# File      : vsc.py
# This file is part of RT-Thread RTOS
# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along
#  with this program; if not, write to the Free Software Foundation, Inc.,
#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Change Logs:
# Date           Author       Notes
# 2018-05-30     Bernard      The first version

"""
Utils for VSCode
"""

import os
import json
import utils
import rtconfig

def GenerateCFiles(env):
    """
    Generate c_cpp_properties files
    """
    if not os.path.exists('.vscode'):
        os.mkdir('.vscode')

    vsc_file = open('.vscode/c_cpp_properties.json', 'w')
    if vsc_file:
        info = utils.ProjectInfo(env)

        cc = os.path.join(rtconfig.EXEC_PATH, rtconfig.CC)
        cc = os.path.abspath(cc).replace('\\', '/')

        config_obj = {}
        config_obj['name'] = 'Win32'
        config_obj['defines'] = info['CPPDEFINES']
        config_obj['intelliSenseMode'] = 'clang-x64'
        config_obj['compilerPath'] = cc
        config_obj['cStandard'] = "c99"
        config_obj['cppStandard'] = "c++11"

        # format "a/b," to a/b. remove first quotation mark("),and remove end (",)
        includePath = []
        for i in info['CPPPATH']:
            if i[0] == '\"' and i[len(i) - 2:len(i)] == '\",':
                includePath.append(i[1:len(i) - 2])
            else:
                includePath.append(i)
        config_obj['includePath'] = includePath

        json_obj = {}
        json_obj['configurations'] = [config_obj]

        vsc_file.write(json.dumps(json_obj, indent=4))
        vsc_file.close()

    return

def GenerateVSCode(env):
    print('Update setting files for VSCode...'),
    GenerateCFiles(env)
    print('Done!')

    return
<commit_msg>[tools][vsc] Support Chinese path<commit_after>#
# File      : vsc.py
# This file is part of RT-Thread RTOS
# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along
#  with this program; if not, write to the Free Software Foundation, Inc.,
#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Change Logs:
# Date           Author       Notes
# 2018-05-30     Bernard      The first version

"""
Utils for VSCode
"""

import os
import json
import utils
import rtconfig

def GenerateCFiles(env):
    """
    Generate c_cpp_properties files
    """
    if not os.path.exists('.vscode'):
        os.mkdir('.vscode')

    vsc_file = open('.vscode/c_cpp_properties.json', 'w')
    if vsc_file:
        info = utils.ProjectInfo(env)

        cc = os.path.join(rtconfig.EXEC_PATH, rtconfig.CC)
        cc = os.path.abspath(cc).replace('\\', '/')

        config_obj = {}
        config_obj['name'] = 'Win32'
        config_obj['defines'] = info['CPPDEFINES']
        config_obj['intelliSenseMode'] = 'clang-x64'
        config_obj['compilerPath'] = cc
        config_obj['cStandard'] = "c99"
        config_obj['cppStandard'] = "c++11"

        # format "a/b," to a/b. remove first quotation mark("),and remove end (",)
        includePath = []
        for i in info['CPPPATH']:
            if i[0] == '\"' and i[len(i) - 2:len(i)] == '\",':
                includePath.append(i[1:len(i) - 2])
            else:
                includePath.append(i)
        config_obj['includePath'] = includePath

        json_obj = {}
        json_obj['configurations'] = [config_obj]

        vsc_file.write(json.dumps(json_obj, ensure_ascii=False, indent=4))
        vsc_file.close()

    return

def GenerateVSCode(env):
    print('Update setting files for VSCode...')
    GenerateCFiles(env)
    print('Done!')

    return
<|endoftext|>


<commit_before>from __future__ import absolute_import

import os
import subprocess
import logging

from topik.readers import iter_document_json_stream, iter_documents_folder,  iter_large_json

from topik.tokenizers import SimpleTokenizer, CollocationsTokenizer, EntitiesTokenizer, MixedTokenizer
from topik.vectorizers import CorpusBOW
from topik.models import LDA
from topik.viz import Termite
from topik.utils import to_r_ldavis, generate_csv_output_file

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

def run_topic_modeling(data, format='json_stream', tokenizer='simple', n_topics=10, dir_path='topic_model',
                       termite_plot=True, output_file=False, r_ldavis=False,  prefix_value=None, event_value=None,
                       field=None):
    """Run your data through all topik functionality and save all results to a specified directory.

    Parameters
    ----------
    data: string
        Input data (file or folder).

    format: {'json_stream', 'folder_files'}.
        The format of your data input. Currently available a json stream or a folder containing text files.
        Default is 'json_stream'

    tokenizer: {'simple', 'collocations', 'entities', 'mixed'}
        The type of tokenizer to use. Default is 'simple'.

    n_topics: int
        Number of topics to find in your data

    dir_path: string
        Directory path to store all topic modeling results files. Default is `topic_model`.

    termite_plot: bool
        Generate termite plot of your model if True. Default is True.

    output_file: bool
        Generate a final summary csv file of your results. For each document: text, tokens, lda_probabilities and topic.

    r_ldavis: bool
        Generate an interactive data visualization of your topics. Default is False.

    prefix_value: string
        For 'large json' format reader, the prefix value to parse.

    event_value: string
        For 'large json' format reader, the event value to parse.

    field: string
        For 'json_stream' data, the field to parse.

    """
    if format == 'folder_files':
        documents = iter_documents_folder(data)
    elif format == 'large_json':
        documents = iter_large_json(data, prefix_value, event_value)
    else:
        documents = iter_document_json_stream(data, field)

    if tokenizer == 'simple':
        corpus = SimpleTokenizer(documents)
    elif tokenizer == 'collocations' :
        corpus = CollocationsTokenizer(documents)
    elif tokenizer == 'entities':
        corpus = EntitiesTokenizer(documents)
    elif tokenizer == 'mixed':
        corpus = MixedTokenizer(documents)
    else:
        print("Processing value invalid, using 1-Simple by default")
        corpus = SimpleTokenizer(documents)

    if os.path.isfile(dir_path):
        pass
    else:
        os.makedirs(dir_path)

    # Create dictionary
    corpus_bow = CorpusBOW(corpus)
    corpus_dict = corpus_bow.save_dict(os.path.join(dir_path, 'corpus.dict'))
    # Serialize and store the corpus
    corpus_file = corpus_bow.serialize(os.path.join(dir_path, 'corpus.mm'))
    # Create LDA model from corpus and dictionary
    lda = LDA(os.path.join(dir_path, 'corpus.mm'), os.path.join(dir_path,'corpus.dict'), n_topics)
    # Generate the input for the termite plot
    lda.termite_data(os.path.join(dir_path,'termite.csv'))
    # Get termite plot for this model
    if termite_plot:
        termite = Termite(os.path.join(dir_path,'termite.csv'), "Termite Plot")
        termite.plot(os.path.join(dir_path,'termite.html'))

    if output_file:
        df_results = generate_csv_output_file(documents, corpus, corpus_bow, lda.model)

    #WIP
    if r_ldavis:
        to_r_ldavis(corpus_bow, dir_name=os.path.join(dir_path, 'ldavis'), lda=lda)
    #    os.chdir(dir_path)
    #    try:
    #        subprocess.call(['R'])
    #    except ValueError:
    #        logging.warning("Unable to run runLDAvis.R")

<commit_msg>stable r_ldavis with simple httpserver<commit_after>from __future__ import absolute_import

import os
import time
import subprocess
import logging
import shutil
import webbrowser
import SimpleHTTPServer
import SocketServer
import threading

import numpy as np

from topik.readers import iter_document_json_stream, iter_documents_folder,  iter_large_json
from topik.tokenizers import SimpleTokenizer, CollocationsTokenizer, EntitiesTokenizer, MixedTokenizer
from topik.vectorizers import CorpusBOW
from topik.models import LDA
from topik.viz import Termite
from topik.utils import to_r_ldavis, generate_csv_output_file

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

BASEDIR = os.path.abspath(os.path.dirname(__file__))

class Handler(SimpleHTTPServer.SimpleHTTPRequestHandler):

    def do_GET(self):
        if self.path == '/':
            self.path = '../topic_model/ldavis/output/index.html'
        return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)

def serve_page():
    PORT = 8000
    handler = SimpleHTTPServer.SimpleHTTPRequestHandler
    httpd = SocketServer.TCPServer(("", PORT), handler)
    print "serving at port", PORT
    httpd.serve_forever()

def run_topic_modeling(data, format='json_stream', tokenizer='simple', n_topics=10, dir_path='./topic_model',
                       termite_plot=True, output_file=False, r_ldavis=False,  prefix_value=None, event_value=None,
                       field=None):
    """Run your data through all topik functionality and save all results to a specified directory.

    Parameters
    ----------
    data: string
        Input data (file or folder).

    format: {'json_stream', 'folder_files'}.
        The format of your data input. Currently available a json stream or a folder containing text files.
        Default is 'json_stream'

    tokenizer: {'simple', 'collocations', 'entities', 'mixed'}
        The type of tokenizer to use. Default is 'simple'.

    n_topics: int
        Number of topics to find in your data

    dir_path: string
        Directory path to store all topic modeling results files. Default is `topic_model`.

    termite_plot: bool
        Generate termite plot of your model if True. Default is True.

    output_file: bool
        Generate a final summary csv file of your results. For each document: text, tokens, lda_probabilities and topic.

    r_ldavis: bool
        Generate an interactive data visualization of your topics. Default is False.

    prefix_value: string
        For 'large json' format reader, the prefix value to parse.

    event_value: string
        For 'large json' format reader, the event value to parse.

    field: string
        For 'json_stream' data, the field to parse.

    """
    np.random.seed(45)

    if format == 'folder_files':
        documents = iter_documents_folder(data)
    elif format == 'large_json':
        documents = iter_large_json(data, prefix_value, event_value)
    else:
        documents = iter_document_json_stream(data, field)

    if tokenizer == 'simple':
        corpus = SimpleTokenizer(documents)
    elif tokenizer == 'collocations' :
        corpus = CollocationsTokenizer(documents)
    elif tokenizer == 'entities':
        corpus = EntitiesTokenizer(documents)
    elif tokenizer == 'mixed':
        corpus = MixedTokenizer(documents)
    else:
        print("Processing value invalid, using 1-Simple by default")
        corpus = SimpleTokenizer(documents)

    if os.path.isdir(dir_path):
        shutil.rmtree(dir_path)

    os.makedirs(dir_path)

    # Create dictionary
    corpus_bow = CorpusBOW(corpus)
    corpus_dict = corpus_bow.save_dict(os.path.join(dir_path, 'corpus.dict'))
    # Serialize and store the corpus
    corpus_file = corpus_bow.serialize(os.path.join(dir_path, 'corpus.mm'))
    # Create LDA model from corpus and dictionary
    lda = LDA(os.path.join(dir_path, 'corpus.mm'), os.path.join(dir_path,'corpus.dict'), n_topics)
    # Generate the input for the termite plot
    lda.termite_data(os.path.join(dir_path,'termite.csv'))
    # Get termite plot for this model
    if termite_plot:
        termite = Termite(os.path.join(dir_path,'termite.csv'), "Termite Plot")
        termite.plot(os.path.join(dir_path,'termite.html'))

    if output_file:
        df_results = generate_csv_output_file(documents, corpus, corpus_bow, lda.model)

    #WIP
    if r_ldavis:
        to_r_ldavis(corpus_bow, dir_name=os.path.join(dir_path, 'ldavis'), lda=lda)
        os.environ["LDAVIS_DIR"] = os.path.join(dir_path, 'ldavis')
        #os.chdir(os.path.join(dir_path, 'ldavis'))
        try:
            subprocess.call(['Rscript', os.path.join(BASEDIR,'R/runLDAvis.R')])
        except ValueError:
            logging.warning("Unable to run runLDAvis.R")
        os.chdir(os.path.join(dir_path, 'ldavis', 'output'))
        sp = subprocess.Popen(['python', '-m', 'SimpleHTTPServer', '8000'])
        webbrowser.open_new_tab('127.0.0.1:8000')
        time.sleep(30)
        sp.kill()

<|endoftext|>