<commit_before># TODO instead of using subprocess, we like to use cloudmesh_base.Shell
import subprocess
import pymongo
from pymongo import MongoClient
import datetime

class db:

    database = None
    jobs = None

    # BUG not a good location for  the data /data
    # BUG not a good location for logpath as we run thi sin user mode
    def startMongo(self, db_path="/data/db/", port="27017", log_path="/var/log/mongod"):

        #Create the data path in case it does not already exist
        subprocess.call(["mkdir", "/data"])
        # try:
        #   r = Shell.mkdir("/data")
        # except:
        #   print " could not cretae"
        # BUG / is not a good location for data and will not work on some systems
        subprocess.call(["mkdir", db_path])

        #Deploy the mongoDB
        # use Shell
        subprocess.call(["mongod", "--dbpath", db_path, "--port", port, "--fork", "--logpath", log_path])

        print "MongoDB has been deployed at path " + db_path + " on port " + port + " with log " + log_path

    def stopMongo(self):

        subprocess.call(["mongod", "--shutdown"])

        print "MongoDB has stopped"

    def connect(self, db_name):

        client = MongoClient()

        self.database = client[db_name]
        self.jobs = self.database.jobs

    def insertJob(self, job_name, input_filename="", output_filename="", start_time=str(datetime.datetime.now()), end_time=str(datetime.datetime.now())):

        if self.database is not None:

            job = {"job_name"        : job_name,
                   "input_filename"  : input_filename,
                   "output_filename" : output_filename,
                   "start_time"      : start_time,
                   "end_time"        : end_time
                   }

            job_id = self.jobs.insert_one(job).inserted_id

            return job_id

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def insertJobObject(self, job):

        if self.database is not None:

            job_id = self.jobs.insert_one(job).inserted_id

            return job_id

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def findJobs(self, key_name="", value=""):

        if self.database is not None:

            #Return all jobs if not given search parameters
            if key_name == "" and value == "":
                return self.jobs.find()

            #Return all jobs given the search parameters
            else:
                return self.jobs.find({key_name: value})

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def deleteJobs(self, key_name="", value=""):

        if self.database is not None:

            #Delete all jobs if no key name or value has been given
            if key_name == "" and value == "":
                self.jobs.remove()

            #Delete only records from a query result if key name and value have been given
            else:
                self.jobs.remove({key_name: value})

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def numJobs(self, key_name="", value=""):

        if self.database is not None:

            #Return the count of all jobs if not given search parameters
            if key_name == "" and value == "":
                return self.jobs.count()

            #Return number of results of the query given the search parameters
            else:
                return self.jobs.find(({key_name: value})).count()

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def updateJobEndTime(self, job_id, end_time=str(datetime.datetime.now())):

        if self.database is not None:

            self.jobs.update({"_id" : job_id}, {"$set": {"end_time" : end_time}}, upsert=False)

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1<commit_msg>Update cm_mongodb.py<commit_after># TODO instead of using subprocess, we like to use cloudmesh_base.Shell
import subprocess
import pymongo
from pymongo import MongoClient
import datetime

class db:

    database = None
    jobs = None

    # BUG not a good location for  the data /data
    # BUG not a good location for logpath as we run this in user mode
    def startMongo(self, db_path="/data/db/", port="27017", log_path="/var/log/mongod"):

        #Create the data path and log path in case they do not exist
        self.createPath(db_path, True)
        self.createPath(log_path, False)

        #Deploy the mongoDB
        # use Shell
        subprocess.call(["mongod", "--dbpath", db_path, "--port", port, "--fork", "--logpath", log_path])

        print "MongoDB has been deployed at path " + db_path + " on port " + port + " with log " + log_path

    def stopMongo(self):

        subprocess.call(["mongod", "--shutdown"])

        print "MongoDB has stopped"

    def connect(self, db_name):

        client = MongoClient()

        self.database = client[db_name]
        self.jobs = self.database.jobs

    def insertJob(self, job_name, input_filename="", output_filename="", start_time=str(datetime.datetime.now()), end_time=str(datetime.datetime.now())):

        if self.database is not None:

            job = {"job_name"        : job_name,
                   "input_filename"  : input_filename,
                   "output_filename" : output_filename,
                   "start_time"      : start_time,
                   "end_time"        : end_time
                   }

            job_id = self.jobs.insert_one(job).inserted_id

            return job_id

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def insertJobObject(self, job):

        if self.database is not None:

            job_id = self.jobs.insert_one(job).inserted_id

            return job_id

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def findJobs(self, key_name="", value=""):

        if self.database is not None:

            #Return all jobs if not given search parameters
            if key_name == "" and value == "":
                return self.jobs.find()

            #Return all jobs given the search parameters
            else:
                return self.jobs.find({key_name: value})

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def deleteJobs(self, key_name="", value=""):

        if self.database is not None:

            #Delete all jobs if no key name or value has been given
            if key_name == "" and value == "":
                self.jobs.remove()

            #Delete only records from a query result if key name and value have been given
            else:
                self.jobs.remove({key_name: value})

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def numJobs(self, key_name="", value=""):

        if self.database is not None:

            #Return the count of all jobs if not given search parameters
            if key_name == "" and value == "":
                return self.jobs.count()

            #Return number of results of the query given the search parameters
            else:
                return self.jobs.find(({key_name: value})).count()

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    def updateJobEndTime(self, job_id, end_time=str(datetime.datetime.now())):

        if self.database is not None:

            self.jobs.update({"_id" : job_id}, {"$set": {"end_time" : end_time}}, upsert=False)

        else:

            print "Please connect to a cloudmesh_job before running this function"
            return -1

    #Create a directory and all of its sub directories
    def createPath(self, completePath, lastPathIsADirectory):

        #Split the full path on all slashes
        paths = completePath.split("/")

        #Build the string of the base directory and all sub directories to the current one
        currentPath = ""

        index = 0

        #Loop through all paths
        for path in paths:

            #Path is blank so do not create a directory
            if path == "":

                #Current path is not the last path
                if index != len(paths) - 1:

                    #Add a slash
                    currentPath = currentPath + "/"

            #Path exists, create the path and add it to the string tracking the complete path
            else:

                #Current path is the last path
                if index == len(paths) - 1:

                    #Do not add a slash on the end
                    currentPath = currentPath + path

                else:
                    currentPath = currentPath + path + "/"

                #Only create the directory if the current path is not the last path or
                #   the path is the last path and it has been flagged as a directory
                if index != len(paths) - 1 or (index == len(paths) - 1 and lastPathIsADirectory):

                    #Attempt to create the directories
                    try:
                        error = subprocess.call(["mkdir", currentPath])

                    except:
                        print "The directory " + currentPath + " already exists"

            index += 1
<|endoftext|>


<commit_before>import numpy as np
from numpy.linalg import inv
import vtk
#import vtk.util.numpy_support as converter
from vtk.io import vtkPLYReader
import h5py
from pipeline_config import COLOR_OCTOMAP_H5_FILE, MERGED_CLOUD_FILE,\
        GPS_FILE, EXPORT_START, EXPORT_STEP, EXPORT_NUM, OCTOMAP_SINGLE_FILES,\
        OCTOMAP_H5_FILE, STATIC_VTK_FILE
from VtkRenderer import VtkPointCloud

from GPSTransforms import IMUTransforms
from GPSReader import GPSReader
from Q50_config import LoadParameters

def load_ply(ply_file):
    reader = vtkPLYReader()
    reader.SetFileName(ply_file)
    reader.Update()
    reader.Update()

    ply_mapper = vtk.vtkPolyDataMapper()
    ply_mapper.SetInputConnection(reader.GetOutputPort())

    actor = vtk.vtkActor()
    actor.SetMapper(ply_mapper)
    return actor

def vtk_transform_from_np(np4x4):
    vtk_matrix = vtk.vtkMatrix4x4()
    for r in range(4):
        for c in range(4):
            vtk_matrix.SetElement(r, c, np4x4[r, c])
    transform = vtk.vtkTransform()
    transform.SetMatrix(vtk_matrix)
    return transform

def load_octomap(octomap_h5_file, conf=0.9, wireframe=False):
    h5f = h5py.File(octomap_h5_file, 'r')
    octree_data = h5f['octree'][...]

    octree_data = octree_data[octree_data[:, 4] > conf]
    pts = vtk.vtkPoints()
    #vtk_pt_data = converter.numpy_to_vtk(np.ascontiguousarray(octree_data[:, 0:3]))
    #pts.SetData(vtk_pt_data)

    use_colors = octree_data.shape[1] > 5
    colors = vtk.vtkUnsignedCharArray()
    colors.SetNumberOfComponents(3)
    #color_data = np.ascontiguousarray(octree_data[:, 5:8])
    #colors = converter.numpy_to_vtk(color_data)
    #colors.SetName('ColorArray')
    #polydata.GetPointData().SetActiveScalars('ColorArray')

    for k in range(octree_data.shape[0]):
        pts.InsertNextPoint(*octree_data[k, 0:3])
        if use_colors:
            r = int(octree_data[k, 5])
            g = int(octree_data[k, 6])
            b = int(octree_data[k, 7])
            colors.InsertNextTupleValue((r, g, b))

    polydata = vtk.vtkPolyData()
    polydata.SetPoints(pts)
    if use_colors:
        polydata.GetPointData().SetScalars(colors)

    cube = vtk.vtkCubeSource()
    cube.SetXLength(octree_data[0, 3])
    cube.SetYLength(octree_data[0, 3])
    cube.SetZLength(octree_data[0, 3])

    glyph = vtk.vtkGlyph3D()
    if use_colors:
        glyph.SetColorModeToColorByScalar()
    glyph.SetSourceConnection(cube.GetOutputPort())
    glyph.SetInput(polydata)
    glyph.ScalingOff()
    glyph.Update()

    mapper = vtk.vtkPolyDataMapper()
    mapper.SetInputConnection(glyph.GetOutputPort())

    actor = vtk.vtkActor()
    actor.SetMapper(mapper)
    if wireframe:
        actor.GetProperty().SetRepresentationToWireframe()
        actor.GetProperty().SetLineWidth(1)
        actor.GetProperty().SetOpacity(0.2)
    actor.GetProperty().LightingOff()

    return actor

def load_vtk_cloud(vtk_cloud_file):
    reader = vtk.vtkDataSetReader()
    reader.SetFileName(vtk_cloud_file)
    reader.Update()

    actor = vtk.vtkActor()
    mapper = vtk.vtkPolyDataMapper()
    mapper.SetInputConnection(reader.GetOutputPort())
    actor.SetMapper(mapper)

    return actor

def get_transforms():
    gps_reader = GPSReader(GPS_FILE)
    gps_data = gps_reader.getNumericData()
    imu_transforms = IMUTransforms(gps_data)
    return imu_transforms

def export_scene_vrml(win, output_file):
    writer = vtk.vtkVRMLExporter()
    writer.SetInput(win)
    writer.SetFileName(output_file)
    writer.Write()

class Blockworld:

    def __init__(self):
        self.start = EXPORT_START
        self.end = EXPORT_START + EXPORT_NUM * EXPORT_STEP
        self.step = EXPORT_STEP
        self.count = 0

        self.ren = vtk.vtkRenderer()

        ''' Transforms '''

        self.imu_transforms = get_transforms()
        self.trans_wrt_imu = self.imu_transforms[self.start:self.end:self.step,
                0:3, 3]
        self.params = LoadParameters('q50_4_3_14_params')

        print 'Adding transforms'

        gps_cloud = VtkPointCloud(self.trans_wrt_imu[:, 0:3],
                0 * self.trans_wrt_imu[:, 0])
        self.ren.AddActor(gps_cloud.get_vtk_cloud())

        #print 'Adding octomap'

        #octomap_actor = load_octomap(OCTOMAP_H5_FILE)
        #self.ren.AddActor(octomap_actor)

        print 'Adding point cloud'

        cloud_actor = load_vtk_cloud(STATIC_VTK_FILE)
        self.ren.AddActor(cloud_actor)

        print 'Adding car'

        self.car = load_ply('gtr.ply')
        self.ren.AddActor(self.car)

        print 'Rendering'

        self.ren.ResetCamera()

        self.win = vtk.vtkRenderWindow()
        self.win.AddRenderer(self.ren)
        self.win.SetSize(400, 400)

        self.iren = vtk.vtkRenderWindowInteractor()
        self.iren .SetRenderWindow(self.win)
        mouseInteractor = vtk.vtkInteractorStyleTrackballCamera()
        self.iren.SetInteractorStyle(mouseInteractor)

        self.iren.Initialize()

        # Whether to write video

        self.record = False

        # Set up time

        self.iren.AddObserver('TimerEvent', self.update)
        self.timer = self.iren.CreateRepeatingTimer(100)

        # Add keypress event
        self.iren.AddObserver('KeyPressEvent', self.keyhandler)
        self.mode = 'ahead'

        self.iren.Start()

    def getCameraPosition(self):
        t = self.start + self.step * self.count
        if self.mode == 'ahead':
            position = self.imu_transforms[t, 0:3, 3]
            focal_point = self.imu_transforms[t + self.step, 0:3, 3]
        elif self.mode == 'behind':
            position = self.imu_transforms[t - self.step, 0:3, 3]
            focal_point = self.imu_transforms[t, 0:3, 3]
        elif self.mode == 'above':
            position = self.imu_transforms[t - self.step, 0:3, 3] + np.array([0, 0, 100.0])
            focal_point = self.imu_transforms[t, 0:3, 3]
        elif self.mode == 'passenger':
            # TODO
            pass
        return position, focal_point

    def keyhandler(self, obj, event):
        key = obj.GetKeySym()
        if key == 'a':
            self.mode = 'above'
        elif key == 'b':
            self.mode == 'behind'
        elif key == 'd':
            self.mode = 'ahead'
        elif key == '0':
            self.count = 0
        elif key == 'r':
            if self.record:
                self.closeVideo()
                self.record = False
            else:
                self.startVideo()
                self.record = True
        else:
            pass

    def update(self, iren, event):
        # Transform the car
        imu_transform = self.imu_transforms[self.start + self.step * self.count, :, :]
        transform = vtk_transform_from_np(imu_transform)
        transform.RotateZ(90)
        transform.Translate(-2, 3, -1)
        self.car.SetUserTransform(transform)

        fren = iren.GetRenderWindow().GetRenderers().GetFirstRenderer()
        cam = fren.GetActiveCamera()
        position, focal_point = self.getCameraPosition()
        cam.SetPosition(position)
        cam.SetFocalPoint(focal_point)
        cam.SetViewUp(0, 0, 1)

        iren.GetRenderWindow().Render()

        if self.record:
            self.writeVideo()

        self.count += 1

    def startVideo(self):
        self.win2img = vtk.vtkWindowToImageFilter()
        self.win2img.SetInput(self.win)
        self.videoWriter = vtk.vtkFFMPEGWriter()
        self.videoWriter.SetFileName('/home/zxie/Desktop/blockworld.avi')
        self.videoWriter.SetInputConnection(self.win2img.GetOutputPort())
        self.videoWriter.SetRate(10)  # 10 fps
        self.videoWriter.SetQuality(2)  # Highest
        self.videoWriter.SetBitRate(1000)  # kilobits/s
        self.videoWriter.SetBitRateTolerance(1000)
        self.videoWriter.Start()

    def writeVideo(self):
        self.win2img.Modified()
        self.videoWriter.Write()

    def closeVideo(self):
        self.videoWriter.End()
        self.videoWriter.Delete()
        self.win2img.Delete()

if __name__ == '__main__':
    blockworld = Blockworld()
<commit_msg>integrate radar in blockworld<commit_after>import numpy as np
from numpy.linalg import inv
import vtk
#import vtk.util.numpy_support as converter
from vtk.io import vtkPLYReader
import h5py
from pipeline_config import COLOR_OCTOMAP_H5_FILE, MERGED_CLOUD_FILE,\
        GPS_FILE, EXPORT_START, EXPORT_STEP, EXPORT_NUM, OCTOMAP_SINGLE_FILES,\
        OCTOMAP_H5_FILE, STATIC_VTK_FILE, DYNAMIC_VTK_FILE, MAP_FILE
from VtkRenderer import VtkPointCloud, VtkBoundingBox
from transformations import euler_from_matrix
from RadarTransforms import loadRDRCamMap, loadRDR, calibrateRadarPts

from GPSTransforms import IMUTransforms
from GPSReader import GPSReader
from Q50_config import LoadParameters

def load_ply(ply_file):
    reader = vtkPLYReader()
    reader.SetFileName(ply_file)
    reader.Update()
    reader.Update()

    ply_mapper = vtk.vtkPolyDataMapper()
    ply_mapper.SetInputConnection(reader.GetOutputPort())

    actor = vtk.vtkActor()
    actor.SetMapper(ply_mapper)
    return actor

def vtk_transform_from_np(np4x4):
    vtk_matrix = vtk.vtkMatrix4x4()
    for r in range(4):
        for c in range(4):
            vtk_matrix.SetElement(r, c, np4x4[r, c])
    transform = vtk.vtkTransform()
    transform.SetMatrix(vtk_matrix)
    return transform

def load_octomap(octomap_h5_file, conf=0.9, wireframe=False):
    h5f = h5py.File(octomap_h5_file, 'r')
    octree_data = h5f['octree'][...]

    octree_data = octree_data[octree_data[:, 4] > conf]
    pts = vtk.vtkPoints()
    #vtk_pt_data = converter.numpy_to_vtk(np.ascontiguousarray(octree_data[:, 0:3]))
    #pts.SetData(vtk_pt_data)

    use_colors = octree_data.shape[1] > 5
    colors = vtk.vtkUnsignedCharArray()
    colors.SetNumberOfComponents(3)
    #color_data = np.ascontiguousarray(octree_data[:, 5:8])
    #colors = converter.numpy_to_vtk(color_data)
    #colors.SetName('ColorArray')
    #polydata.GetPointData().SetActiveScalars('ColorArray')

    for k in range(octree_data.shape[0]):
        pts.InsertNextPoint(*octree_data[k, 0:3])
        if use_colors:
            r = int(octree_data[k, 5])
            g = int(octree_data[k, 6])
            b = int(octree_data[k, 7])
            colors.InsertNextTupleValue((r, g, b))

    polydata = vtk.vtkPolyData()
    polydata.SetPoints(pts)
    if use_colors:
        polydata.GetPointData().SetScalars(colors)

    cube = vtk.vtkCubeSource()
    cube.SetXLength(octree_data[0, 3])
    cube.SetYLength(octree_data[0, 3])
    cube.SetZLength(octree_data[0, 3])

    glyph = vtk.vtkGlyph3D()
    if use_colors:
        glyph.SetColorModeToColorByScalar()
    glyph.SetSourceConnection(cube.GetOutputPort())
    glyph.SetInput(polydata)
    glyph.ScalingOff()
    glyph.Update()

    mapper = vtk.vtkPolyDataMapper()
    mapper.SetInputConnection(glyph.GetOutputPort())

    actor = vtk.vtkActor()
    actor.SetMapper(mapper)
    if wireframe:
        actor.GetProperty().SetRepresentationToWireframe()
        actor.GetProperty().SetLineWidth(1)
        actor.GetProperty().SetOpacity(0.2)
    actor.GetProperty().LightingOff()

    return actor

def load_vtk_cloud(vtk_cloud_file):
    reader = vtk.vtkDataSetReader()
    reader.SetFileName(vtk_cloud_file)
    reader.Update()

    actor = vtk.vtkActor()
    mapper = vtk.vtkPolyDataMapper()
    mapper.SetInputConnection(reader.GetOutputPort())
    actor.SetMapper(mapper)

    return actor

def get_transforms():
    gps_reader = GPSReader(GPS_FILE)
    gps_data = gps_reader.getNumericData()
    imu_transforms = IMUTransforms(gps_data)
    return imu_transforms

def export_scene_vrml(win, output_file):
    writer = vtk.vtkVRMLExporter()
    writer.SetInput(win)
    writer.SetFileName(output_file)
    writer.Write()

class Blockworld:

    def __init__(self):
        self.start = EXPORT_START
        self.end = EXPORT_START + EXPORT_NUM * EXPORT_STEP
        self.step = EXPORT_STEP
        self.count = 0

        self.ren = vtk.vtkRenderer()

        ''' Transforms '''

        self.imu_transforms = get_transforms()
        self.trans_wrt_imu = self.imu_transforms[self.start:self.end:self.step,
                0:3, 3]
        self.params = LoadParameters('q50_4_3_14_params')
        self.radar_params = self.params['radar']
        self.lidar_params = self.params['lidar']

        ''' Radar '''

        self.rdr_pts = loadRDRCamMap(MAP_FILE)
        self.radar_actors = []

        print 'Adding transforms'

        gps_cloud = VtkPointCloud(self.trans_wrt_imu[:, 0:3],
                0 * self.trans_wrt_imu[:, 0])
        self.ren.AddActor(gps_cloud.get_vtk_cloud())

        #print 'Adding octomap'

        #octomap_actor = load_octomap(OCTOMAP_H5_FILE)
        #self.ren.AddActor(octomap_actor)

        print 'Adding point cloud'

        cloud_actor = load_vtk_cloud(STATIC_VTK_FILE)
        self.ren.AddActor(cloud_actor)

        #dynamic_actor = load_vtk_cloud(DYNAMIC_VTK_FILE)
        #dynamic_actor.GetProperty().SetColor(0, 0, 1)
        #dynamic_actor.GetMapper().ScalarVisibilityOff()
        #self.ren.AddActor(dynamic_actor)

        print 'Adding car'

        self.car = load_ply('gtr.ply')
        self.ren.AddActor(self.car)

        print 'Rendering'

        self.ren.ResetCamera()

        self.win = vtk.vtkRenderWindow()
        self.win.AddRenderer(self.ren)
        self.win.SetSize(400, 400)

        self.iren = vtk.vtkRenderWindowInteractor()
        self.iren .SetRenderWindow(self.win)
        mouseInteractor = vtk.vtkInteractorStyleTrackballCamera()
        self.iren.SetInteractorStyle(mouseInteractor)

        self.iren.Initialize()

        # Whether to write video

        self.record = False

        # Set up time

        self.iren.AddObserver('TimerEvent', self.update)
        self.timer = self.iren.CreateRepeatingTimer(100)

        # Add keypress event
        self.iren.AddObserver('KeyPressEvent', self.keyhandler)
        self.mode = 'ahead'

        self.iren.Start()

    def getCameraPosition(self):
        t = self.start + self.step * self.count
        if self.mode == 'ahead':
            position = self.imu_transforms[t, 0:3, 3]
            focal_point = self.imu_transforms[t + self.step, 0:3, 3]
        elif self.mode == 'behind':
            # FIXME Tune this
            position = self.imu_transforms[t - 5*self.step, 0:3, 3]
            focal_point = self.imu_transforms[t - 4*self.step, 0:3, 3]
        elif self.mode == 'above':
            position = self.imu_transforms[t - self.step, 0:3, 3] + np.array([0, 0, 200.0])
            focal_point = self.imu_transforms[t, 0:3, 3]
        elif self.mode == 'passenger':
            # TODO Not sure being inside mesh works...
            pass
        return position, focal_point

    def keyhandler(self, obj, event):
        key = obj.GetKeySym()
        if key == 'a':
            self.mode = 'above'
        elif key == 'b':
            self.mode = 'behind'
        elif key == 'd':
            self.mode = 'ahead'
        elif key == '0':
            self.count = 0
        elif key == 'r':
            if self.record:
                self.closeVideo()
                self.record = False
            else:
                self.startVideo()
                self.record = True
        else:
            pass

    def updateRadar(self):
        # Taken from testDrawRadarOnMap.py
        fren = self.iren.GetRenderWindow().GetRenderers().GetFirstRenderer()
        t = self.start + self.step * self.count
        radar_data = loadRDR(self.rdr_pts[t])[0]

        if radar_data.shape[0] > 0:
            #Convert from radar to lidar ref-frame
            radar_data[:, :3] = calibrateRadarPts(radar_data[:, :3], self.radar_params)

            #Convert from lidar to IMU ref-frame
            radar_data[:, :3] = np.dot(self.lidar_params['T_from_l_to_i'][:3, :3],
                radar_data[:, :3].transpose()).transpose()

            h_radar_data = np.hstack((radar_data[:, :3], np.ones((radar_data.shape[0], 1))))

            radar_data[:, :3] = np.dot(self.imu_transforms[t],
                h_radar_data.transpose()).transpose()[:, :3]

            for i in xrange(len(self.radar_actors)):
                fren.RemoveActor(self.radar_actors[i])

            self.radar_actors = []
            self.radar_clouds = []

            for i in xrange(radar_data.shape[0]):
                self.radar_clouds.append(VtkBoundingBox(radar_data[i, :]))
                (ax, ay, az) = euler_from_matrix(self.imu_transforms[t])
                box = self.radar_clouds[i].get_vtk_box(rot=az*180/np.pi)
                self.radar_actors.append(box)
                fren.AddActor(self.radar_actors[i])

    def update(self, iren, event):
        # Transform the car

        t = self.start + self.step * self.count
        imu_transform = self.imu_transforms[t, :, :]
        transform = vtk_transform_from_np(imu_transform)
        transform.RotateZ(90)
        transform.Translate(-2, -3, -2)
        self.car.SetUserTransform(transform)

        # Add the radar

        self.updateRadar()

        # Set camera position
        fren = iren.GetRenderWindow().GetRenderers().GetFirstRenderer()
        cam = fren.GetActiveCamera()
        position, focal_point = self.getCameraPosition()
        cam.SetPosition(position)
        cam.SetFocalPoint(focal_point)
        cam.SetViewUp(0, 0, 1)

        iren.GetRenderWindow().Render()

        if self.record:
            self.writeVideo()

        self.count += 1

    def startVideo(self):
        self.win2img = vtk.vtkWindowToImageFilter()
        self.win2img.SetInput(self.win)
        self.videoWriter = vtk.vtkFFMPEGWriter()
        self.videoWriter.SetFileName('/home/zxie/Desktop/blockworld.avi')
        self.videoWriter.SetInputConnection(self.win2img.GetOutputPort())
        self.videoWriter.SetRate(10)  # 10 fps
        self.videoWriter.SetQuality(2)  # Highest
        self.videoWriter.SetBitRate(1000)  # kilobits/s
        self.videoWriter.SetBitRateTolerance(1000)
        self.videoWriter.Start()

    def writeVideo(self):
        self.win2img.Modified()
        self.videoWriter.Write()

    def closeVideo(self):
        self.videoWriter.End()
        self.videoWriter.Delete()
        self.win2img.Delete()

if __name__ == '__main__':
    blockworld = Blockworld()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-

# Spectrum Simulations
# To determine the recovery of planetary spectra.

# imports
from __future__ import division, print_function
# from todcor import todcor
# from todcor import create_cross_correlations
from astropy.io import fits
from spectrum_overload.Spectrum import Spectrum
import numpy as np
import matplotlib.pyplot as plt
import copy
import os

from simulation_utilities import spectrum_plotter

def RV_shift():
    """ Doppler shift spectrum"""
    pass

def log_RV_shift():
    """ Doppler shift when log-linear spectrum is used"""
    pass

from simulation_utilities import add_noise
# def add_noise(spectra, snr):
#    """Add noise to spectrum at the level given by snr"""

def combine_spectra(star, planet, alpha):
    """"Combine the Spectrum objects "star" and "planet" with a strength ratio of aplha
    spec = star + planet * alpha

    """
    star = copy.copy(star)
    planet = copy.copy(planet)

    if np.all(star.xaxis == planet.xaxis):   # make sure wavelenghts even first
        pass
    else:
        planet.spline_interpolate_to(star)
    # combined_spectrum = star + (planet*alpha)
    # Combined spectra with proper normalization
    norm_factor = 1 / (1 + alpha)
    combined_spectrum = (star + (planet*alpha)) * norm_factor

    return combined_spectrum

def simple_normalization(spectrum):
    """ Simple normalization of pheonix spectra """
    from astropy.modeling import models, fitting
    p1 = models.Polynomial1D(1)
    p1.c0 = spectrum.flux[0]
    p1.c1 = ((spectrum.flux[-1]-spectrum.flux[0]) /
             (spectrum.xaxis[-1]-spectrum.xaxis[0]))

    #print("Printing p1", p1)
    pfit = fitting.LinearLSQFitter()
    new_model = pfit(p1, spectrum.xaxis, spectrum.flux)
    #print(new_model)
    fit_norm = new_model(spectrum.xaxis)
    norm_spectrum = spectrum / fit_norm
    flux = norm_spectrum.flux

    # Normalization (use first 50 points below 1.2 as continuum)
    maxes = flux[(flux < 1.2)].argsort()[-50:][::-1]
    norm_spectrum = norm_spectrum / np.median(flux[maxes])

    return norm_spectrum
    # Split spectrum into a bunch, then fit along it along the top

    # length = len(spectrum)
    # indexes = range(0, length, 100)
    # This is bad coding I am just trying to get something together
    # maxes = []
    # for index1, index2 in zip(indexes[:-1], indexes[1:]):
#        section = np.argmax(spectrum[index1:index2])
#        m = argmax(section)
#        maxes.append(m)
#    print("maxes", maxes)
#    print("indixes", indexes)

def load_PHOENIX_hd30501(limits=None, normalize=False):
    """ Load in the phoenix spectra of HD30501 and HD30501b

    Returns:
    w_mod : Wavelength in nm
    I_star : stellar intensity
    I_bd : companion intensity
    hdr_star : Star header
    hdr_bd : Companion header
    """

    pathwave = "/home/jneal/Phd/data/phoenixmodels/" \
               "WAVE_PHOENIX-ACES-AGSS-COND-2011.fits"
    bd_model = "/home/jneal/Phd/data/phoenixmodels/" \
               "HD30501b-lte02500-5.00-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits"
    star_model = "/home/jneal/Phd/data/phoenixmodels/" \
                 "HD30501-lte05200-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits"

    I_bdmod = fits.getdata(bd_model)
    I_star = fits.getdata(star_model)
    hdr_bd = fits.getheader(bd_model)
    hdr_star = fits.getheader(star_model)
    w_mod = fits.getdata(pathwave)

    w_mod /= 10   # turn into nm

    if limits:
        """ Apply wavelength limits with slicing"""
        mask = (w_mod > limits[0]) & (w_mod < limits[1])
        w_mod = w_mod[mask]
        I_star = I_star[mask]
        I_bdmod = I_bdmod[mask]

    if normalize:
        """ Apply normalization to loaded spectrum"""
        if limits is None:
            print("Warning! Limits should be given when using normalization")
        star_spec = Spectrum(flux=I_star, xaxis=w_mod)
        result = simple_normalization(star_spec)
        I_star = result.flux
        bd_spec = Spectrum(flux=I_bdmod, xaxis=w_mod)
        result = simple_normalization(bd_spec)
        I_bdmod = result.flux

    return w_mod, I_star, I_bdmod, hdr_star, hdr_bd

def main():
    # Load in the pheonix spectra
    pathwave = "/home/jneal/Phd/data/phoenixmodels/" \
               "WAVE_PHOENIX-ACES-AGSS-COND-2011.fits"
    bd_model = "/home/jneal/Phd/data/phoenixmodels/" \
               "HD30501b-lte02500-5.00-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits"
    star_model = "/home/jneal/Phd/data/phoenixmodels/" \
                 "HD30501-lte05200-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits"

    I_bdmod = fits.getdata(bd_model)
    I_star = fits.getdata(star_model)
    hdr_bd = fits.getheader(bd_model)
    hdr_star = fits.getheader(star_model)
    w_mod = fits.getdata(pathwave)

    w_mod /= 10   # turn into nm

    star_spec = Spectrum(xaxis=w_mod, flux=I_star, calibrated=True)
    bd_spec = Spectrum(xaxis=w_mod, flux=I_bdmod, calibrated=True)

    # Wavelength selection from 2100-2200nm
    star_spec.wav_select(2100, 2200)
    bd_spec.wav_select(2100, 2200)
    # star_spec.wav_select(100, 3000)
    # bd_spec.wav_select(100, 3000)

    star_spec = simple_normalization(star_spec)
    bd_spec = simple_normalization(bd_spec)

    # Loop over different things
    print(star_spec.xaxis)
    print(bd_spec.xaxis)
    combined = combine_spectra(star_spec, bd_spec, alpha=0.01)
    combined2 = combine_spectra(star_spec, bd_spec, alpha=0.1)

    spectrum_plotter(bd_spec, label="HD30501b")
    plt.title("Pheonix Spectra HD30501A and HD30501b")
    plt.ylabel("Flux [ergs/s/cm^2/cm]")
    # plt.ylabel("Normalized Flux")
    plt.xlabel("Wavelength [nm]")
    spectrum_plotter(star_spec + 1, label="HD30501A")

    # plotter(combined2)
    spectrum_plotter(combined + 1, label="Combined", show=True)

    # Why does this plot things twice???
    # Add rv attributes to spectrum for TODCOR
    # Don't yet know what the parameters are though
    # bd_spec.rv = 10
    # star_spec.rv = 10

    RV_shift = np.arange(0, 20, 3)
    alpha = 0.005
    # alpha = 0.01
    for RV in RV_shift:
        """ """
        bd_shifted = copy.copy(bd_spec)
        # RV shift BD spectra
        bd_shifted.doppler_shift(RV)

        combined_spec = combine_spectra(star_spec, bd_shifted, alpha=alpha)

        combined_spec.wav_select(2158, 2159)
        spectrum_plotter(combined_spec + (RV/200), label="RV shift={} km/s".format(RV))

    plt.title("Combined spectra (star+alpha*planet), alpha = {}".format(alpha))
    plt.legend(loc=0)
    # , bbox_to_anchor=(1.4, 0.9), ncol=1, fancybox=True, shadow=True)
    plt.ylabel("Norm Flux")
    plt.xlabel("Wavelength (nm)")
    plt.show()

    # Try run todoc stuff

    # Probably need to change my formating into ajriddles format to use his code.
    # ccf1,ccf2,ccf12,images = create_cross_correlations(combined2, star_spec, bd_spec)

    # plt.plot(ccf1)
    # plt.plot(ccf2)
    # plt.plot(ccf12)
    # plt.show()
    # pshift = 5     #????
    # sshift = 0    #????
    # cps_R,cps_m,cps_fits,cps_vp,cps_vs,cps_cntr = todcor(ccf1,ccf2,ccf12,pshift,sshift,images)

# main()

if __name__ == "__main__":
    main()
<commit_msg>format fixes<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-

# Spectrum Simulations
# To determine the recovery of planetary spectra.

# imports
from __future__ import division, print_function
# from todcor import todcor
# from todcor import create_cross_correlations
from astropy.io import fits
from spectrum_overload.Spectrum import Spectrum
import numpy as np
import matplotlib.pyplot as plt
import copy
import os

from simulation_utilities import spectrum_plotter

def RV_shift():
    """ Doppler shift spectrum"""
    pass

def log_RV_shift():
    """ Doppler shift when log-linear spectrum is used"""
    pass

from simulation_utilities import add_noise
# def add_noise(spectra, snr):
#    """Add noise to spectrum at the level given by snr"""

def combine_spectra(star, planet, alpha):
    """"Combine the Spectrum objects "star" and "planet" with a strength ratio of aplha
    spec = star + planet * alpha

    """
    star = copy.copy(star)
    planet = copy.copy(planet)

    if np.all(star.xaxis == planet.xaxis):   # make sure wavelenghts even first
        pass
    else:
        planet.spline_interpolate_to(star)
    # combined_spectrum = star + (planet*alpha)
    # Combined spectra with proper normalization
    norm_factor = 1 / (1 + alpha)
    combined_spectrum = (star + (planet * alpha)) * norm_factor

    return combined_spectrum

def simple_normalization(spectrum):
    """ Simple normalization of pheonix spectra """
    from astropy.modeling import models, fitting
    p1 = models.Polynomial1D(1)
    p1.c0 = spectrum.flux[0]
    p1.c1 = ((spectrum.flux[-1] - spectrum.flux[0]) / (spectrum.xaxis[-1] - spectrum.xaxis[0]))

    # print("Printing p1", p1)
    pfit = fitting.LinearLSQFitter()
    new_model = pfit(p1, spectrum.xaxis, spectrum.flux)
    # print(new_model)
    fit_norm = new_model(spectrum.xaxis)
    norm_spectrum = spectrum / fit_norm
    flux = norm_spectrum.flux

    # Normalization (use first 50 points below 1.2 as continuum)
    maxes = flux[(flux < 1.2)].argsort()[-50:][::-1]
    norm_spectrum = norm_spectrum / np.median(flux[maxes])

    return norm_spectrum
    # Split spectrum into a bunch, then fit along it along the top

    # length = len(spectrum)
    # indexes = range(0, length, 100)
    # This is bad coding I am just trying to get something together
    # maxes = []
    # for index1, index2 in zip(indexes[:-1], indexes[1:]):
#        section = np.argmax(spectrum[index1:index2])
#        m = argmax(section)
#        maxes.append(m)
#    print("maxes", maxes)
#    print("indixes", indexes)

def load_PHOENIX_hd30501(limits=None, normalize=False):
    """ Load in the phoenix spectra of HD30501 and HD30501b

    Returns:
    w_mod : Wavelength in nm
    I_star : stellar intensity
    I_bd : companion intensity
    hdr_star : Star header
    hdr_bd : Companion header
    """

    pathwave = "/home/jneal/Phd/data/phoenixmodels/" \
               "WAVE_PHOENIX-ACES-AGSS-COND-2011.fits"
    bd_model = "/home/jneal/Phd/data/phoenixmodels/" \
               "HD30501b-lte02500-5.00-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits"
    star_model = "/home/jneal/Phd/data/phoenixmodels/" \
                 "HD30501-lte05200-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits"

    I_bdmod = fits.getdata(bd_model)
    I_star = fits.getdata(star_model)
    hdr_bd = fits.getheader(bd_model)
    hdr_star = fits.getheader(star_model)
    w_mod = fits.getdata(pathwave)

    w_mod /= 10   # turn into nm

    if limits:
        """ Apply wavelength limits with slicing"""
        mask = (w_mod > limits[0]) & (w_mod < limits[1])
        w_mod = w_mod[mask]
        I_star = I_star[mask]
        I_bdmod = I_bdmod[mask]

    if normalize:
        """ Apply normalization to loaded spectrum"""
        if limits is None:
            print("Warning! Limits should be given when using normalization")
        star_spec = Spectrum(flux=I_star, xaxis=w_mod)
        result = simple_normalization(star_spec)
        I_star = result.flux
        bd_spec = Spectrum(flux=I_bdmod, xaxis=w_mod)
        result = simple_normalization(bd_spec)
        I_bdmod = result.flux

    return w_mod, I_star, I_bdmod, hdr_star, hdr_bd

def main():
    # Load in the pheonix spectra
    pathwave = "/home/jneal/Phd/data/phoenixmodels/" \
               "WAVE_PHOENIX-ACES-AGSS-COND-2011.fits"
    bd_model = "/home/jneal/Phd/data/phoenixmodels/" \
               "HD30501b-lte02500-5.00-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits"
    star_model = "/home/jneal/Phd/data/phoenixmodels/" \
                 "HD30501-lte05200-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits"

    I_bdmod = fits.getdata(bd_model)
    I_star = fits.getdata(star_model)
    hdr_bd = fits.getheader(bd_model)
    hdr_star = fits.getheader(star_model)
    w_mod = fits.getdata(pathwave)

    w_mod /= 10   # turn into nm

    star_spec = Spectrum(xaxis=w_mod, flux=I_star, calibrated=True)
    bd_spec = Spectrum(xaxis=w_mod, flux=I_bdmod, calibrated=True)

    # Wavelength selection from 2100-2200nm
    star_spec.wav_select(2100, 2200)
    bd_spec.wav_select(2100, 2200)
    # star_spec.wav_select(100, 3000)
    # bd_spec.wav_select(100, 3000)

    star_spec = simple_normalization(star_spec)
    bd_spec = simple_normalization(bd_spec)

    # Loop over different things
    print(star_spec.xaxis)
    print(bd_spec.xaxis)
    combined = combine_spectra(star_spec, bd_spec, alpha=0.01)
    combined2 = combine_spectra(star_spec, bd_spec, alpha=0.1)

    spectrum_plotter(bd_spec, label="HD30501b")
    plt.title("Pheonix Spectra HD30501A and HD30501b")
    plt.ylabel("Flux [ergs/s/cm^2/cm]")
    # plt.ylabel("Normalized Flux")
    plt.xlabel("Wavelength [nm]")
    spectrum_plotter(star_spec + 1, label="HD30501A")

    # plotter(combined2)
    spectrum_plotter(combined + 1, label="Combined", show=True)

    # Why does this plot things twice???
    # Add rv attributes to spectrum for TODCOR
    # Don't yet know what the parameters are though
    # bd_spec.rv = 10
    # star_spec.rv = 10

    RV_shift = np.arange(0, 20, 3)
    alpha = 0.005
    # alpha = 0.01
    for RV in RV_shift:
        """ """
        bd_shifted = copy.copy(bd_spec)
        # RV shift BD spectra
        bd_shifted.doppler_shift(RV)

        combined_spec = combine_spectra(star_spec, bd_shifted, alpha=alpha)

        combined_spec.wav_select(2158, 2159)
        spectrum_plotter(combined_spec + (RV/200), label="RV shift={} km/s".format(RV))

    plt.title("Combined spectra (star+alpha*planet), alpha = {}".format(alpha))
    plt.legend(loc=0)
    # , bbox_to_anchor=(1.4, 0.9), ncol=1, fancybox=True, shadow=True)
    plt.ylabel("Norm Flux")
    plt.xlabel("Wavelength (nm)")
    plt.show()

    # Try run todoc stuff

    # Probably need to change my formating into ajriddles format to use his code.
    # ccf1,ccf2,ccf12,images = create_cross_correlations(combined2, star_spec, bd_spec)

    # plt.plot(ccf1)
    # plt.plot(ccf2)
    # plt.plot(ccf12)
    # plt.show()
    # pshift = 5     #????
    # sshift = 0    #????
    # cps_R,cps_m,cps_fits,cps_vp,cps_vs,cps_cntr = todcor(ccf1,ccf2,ccf12,pshift,sshift,images)

# main()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os, re, sys
import datetime, logging

from picklable_itertools.extras import partition_all
from concurrent.futures import ThreadPoolExecutor
from dateutil.parser import parse as date_parse
import requests

import pyexcel.ext.xlsx
import pyexcel as pe
import indicoio

from .client import ESConnection
from .schema import Document
from .summary import Summary

# turn down requests log verbosity
logging.getLogger('requests').setLevel(logging.CRITICAL)

# Processing
EXECUTOR = ThreadPoolExecutor(max_workers=8)
ENGLISH_SUMMARIZER = Summary(language="english")
NOW = datetime.datetime.now()
ONE_YEAR_AGO = NOW.replace(year = NOW.year - 1)
DESCRIPTION_THRESHOLD = 600

# Logging
root = logging.getLogger()
root.setLevel(logging.DEBUG)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
root.addHandler(ch)

# Indico
indicoio.config.cloud = 'themeextraction'

with open('cluster/search/sp500.txt') as f:
    SP_TICKERS = f.read().splitlines()

def _not_in_sp500(document):
    in_text = any([re.compile(r'[^a-zA-Z]'+ re.escape(ticker)+ r'[^a-zA-Z]').search(document["text"]) for ticker in SP_TICKERS])
    in_title = any([re.compile(r'[^a-zA-Z]'+ re.escape(ticker)+ r'[^a-zA-Z]').search(document["title"]) for ticker in SP_TICKERS])
    return not(in_text or in_title)

def _relevant_and_recent(document):
    if not document.link or "Service Unavailable" in document.text:
        return False
    if not document["text"] or not document["title"]:
        return False
    if date_parse(document.get("date")) < ONE_YEAR_AGO and _not_in_sp500(document):
        return False
    return True

def parse_obj_to_document(obj):
    text = obj.get("description_text")
    title = obj.get("name_post", obj.get("name_story"))
    link = obj.get("url_post", obj.get("url_story"))
    tags = [obj.get("name_categorie", obj.get("name_topics"))]
    pub_date = obj.get("date_published", obj.get("date_published_story"))

    # Remove CSS & HTML residue
    text = re.sub(r"<[^>]*>", "", text)
    text = re.sub(r"[\s]+", " ", text)
    text = re.sub(r"[[\w\-]*\s*[\.[\w\-]*\s*]*]*\s*{[^}]*}", "", text)
    text = text.encode("ascii", "ignore").decode("ascii", "ignore")

    return Document(
        title=title,
        text=text,
        link=link,
        tags=tags,
        length=len(text),
        date=pub_date,
        indico={}
    )

def add_indico(documents):
    documents = filter(_relevant_and_recent, documents)
    try:
        analysis = {}
        summaries = [EXECUTOR.submit(ENGLISH_SUMMARIZER.parse, doc.get("text"), sentences=3) for doc in documents]

        analysis["title_sentiment_hq"] = EXECUTOR.submit(indicoio.sentiment_hq, [doc.get("title") for doc in documents])
        analysis["title_keywords"] = EXECUTOR.submit(indicoio.keywords, [doc.get("title") for doc in documents], version=1)
        analysis["title_ner"] = EXECUTOR.submit(indicoio.named_entities, [doc.get("title") for doc in documents], version=2)
        analysis["sentiment_hq"] = EXECUTOR.submit(indicoio.sentiment_hq, [doc.get("text") for doc in documents])
        analysis["keywords"] = EXECUTOR.submit(indicoio.keywords, [doc.get("text") for doc in documents], version=1)
        analysis["ner"] = EXECUTOR.submit(indicoio.named_entities, [doc.get("text") for doc in documents], version=2)

        analysis["title_sentiment_hq"] = analysis["title_sentiment_hq"].result()
        analysis["title_keywords"] = analysis["title_keywords"].result()
        analysis["title_ner"] = analysis["title_ner"].result()
        analysis["sentiment_hq"] = analysis["sentiment_hq"].result()
        analysis["keywords"] = analysis["keywords"].result()
        analysis["ner"] = analysis["ner"].result()

        for i in xrange(len(documents)):
            # Title Analysis
            documents[i]["indico"] = {}
            documents[i]["indico"]["title_sentiment"] = analysis.get('title_sentiment_hq')[i]
            documents[i]["indico"]["title_keywords"] = analysis.get('title_keywords')[i]
            documents[i]["indico"]["title_people"] = analysis.get('title_ner')[i].get("people")
            documents[i]["indico"]["title_places"] = analysis.get('title_ner')[i].get("places")
            documents[i]["indico"]["title_organizations"] = analysis.get('title_ner')[i].get("organizations")

            # Main Text
            documents[i]["indico"]["sentiment"] = analysis.get('sentiment_hq')[i]
            documents[i]["indico"]["keywords"] = analysis.get('keywords')[i]
            documents[i]["indico"]["people"] = analysis.get('ner')[i].get("people")
            documents[i]["indico"]["places"] = analysis.get('ner')[i].get("places")
            documents[i]["indico"]["organizations"] = analysis.get('ner')[i].get("organizations")

            # Summary
            documents[i]["summary"] = summaries[i].result()

        return documents
    except:
        import traceback; traceback.print_exc()
        length = len(documents)/2
        if length <= 1:
            return []
        return add_indico(documents[:length]) + add_indico(documents[length:])

def get_all_data_files(current_dir):
    all_files = []
    for subdir, dirs, files in os.walk(current_dir):
        for filename in files:
            all_files.append(os.path.join(current_dir, filename))
    return all_files

def read_data_file(data_file):
    try:
        lines = [line for line in pe.get_records(file_name=data_file, streaming=True) if len(line.get("description_text", "")) > DESCRIPTION_THRESHOLD][:5]
        root.debug("Parsing Documents for {0}".format(data_file))
        documents = map(lambda x: parse_obj_to_document(x), lines)
        return documents
    except:
        import traceback; traceback.print_exc()
        return []

def upload_data(es, data_file):
    root.debug("Beginning Processing for {0}".format(data_file))
    all_documents = read_data_file(data_file)
    root.debug("Read Data for {0}".format(data_file))
    for documents in partition_all(20, all_documents):
        root.debug("Adding Indico for {0}".format(data_file))
        documents = add_indico(documents)
        root.debug("Uploading to elasticsearch for {0}".format(data_file))
        es.upload(documents)

if __name__ == "__main__":
    executor = ThreadPoolExecutor(max_workers=4)
    es = ESConnection("localhost:9200")
    directory = os.path.join(os.path.dirname(__file__), '../../inputxl')
    files = get_all_data_files(directory)

    futures = []
    for _file in files:
        futures.append(executor.submit(upload_data, es, _file))
    map(lambda x: x.result(), futures)
<commit_msg>REMOVE: 5 limit on doc<commit_after>import os, re, sys
import datetime, logging

from picklable_itertools.extras import partition_all
from concurrent.futures import ThreadPoolExecutor
from dateutil.parser import parse as date_parse
import requests

import pyexcel.ext.xlsx
import pyexcel as pe
import indicoio

from .client import ESConnection
from .schema import Document
from .summary import Summary

# Turn down requests log verbosity
logging.getLogger('requests').setLevel(logging.CRITICAL)

# Processing
EXECUTOR = ThreadPoolExecutor(max_workers=8)
ENGLISH_SUMMARIZER = Summary(language="english")
NOW = datetime.datetime.now()
ONE_YEAR_AGO = NOW.replace(year = NOW.year - 1)
DESCRIPTION_THRESHOLD = 600

# Logging
root = logging.getLogger()
root.setLevel(logging.DEBUG)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
root.addHandler(ch)

# Indico
indicoio.config.cloud = 'themeextraction'

with open('cluster/search/sp500.txt') as f:
    SP_TICKERS = f.read().splitlines()

def _not_in_sp500(document):
    in_text = any([re.compile(r'[^a-zA-Z]'+ re.escape(ticker)+ r'[^a-zA-Z]').search(document["text"]) for ticker in SP_TICKERS])
    in_title = any([re.compile(r'[^a-zA-Z]'+ re.escape(ticker)+ r'[^a-zA-Z]').search(document["title"]) for ticker in SP_TICKERS])
    return not(in_text or in_title)

def _relevant_and_recent(document):
    if not document.link or "Service Unavailable" in document.text:
        return False
    if not document["text"] or not document["title"]:
        return False
    if date_parse(document.get("date")) < ONE_YEAR_AGO and _not_in_sp500(document):
        return False
    return True

def parse_obj_to_document(obj):
    text = obj.get("description_text")
    title = obj.get("name_post", obj.get("name_story"))
    link = obj.get("url_post", obj.get("url_story"))
    tags = [obj.get("name_categorie", obj.get("name_topics"))]
    pub_date = obj.get("date_published", obj.get("date_published_story"))

    # Remove CSS & HTML residue
    text = re.sub(r"<[^>]*>", "", text)
    text = re.sub(r"[\s]+", " ", text)
    text = re.sub(r"[[\w\-]*\s*[\.[\w\-]*\s*]*]*\s*{[^}]*}", "", text)
    text = text.encode("ascii", "ignore").decode("ascii", "ignore")

    return Document(
        title=title,
        text=text,
        link=link,
        tags=tags,
        length=len(text),
        date=pub_date,
        indico={}
    )

def add_indico(documents):
    documents = filter(_relevant_and_recent, documents)
    try:
        analysis = {}
        summaries = [EXECUTOR.submit(ENGLISH_SUMMARIZER.parse, doc.get("text"), sentences=3) for doc in documents]

        analysis["title_sentiment_hq"] = EXECUTOR.submit(indicoio.sentiment_hq, [doc.get("title") for doc in documents])
        analysis["title_keywords"] = EXECUTOR.submit(indicoio.keywords, [doc.get("title") for doc in documents], version=1)
        analysis["title_ner"] = EXECUTOR.submit(indicoio.named_entities, [doc.get("title") for doc in documents], version=2)
        analysis["sentiment_hq"] = EXECUTOR.submit(indicoio.sentiment_hq, [doc.get("text") for doc in documents])
        analysis["keywords"] = EXECUTOR.submit(indicoio.keywords, [doc.get("text") for doc in documents], version=1)
        analysis["ner"] = EXECUTOR.submit(indicoio.named_entities, [doc.get("text") for doc in documents], version=2)

        analysis["title_sentiment_hq"] = analysis["title_sentiment_hq"].result()
        analysis["title_keywords"] = analysis["title_keywords"].result()
        analysis["title_ner"] = analysis["title_ner"].result()
        analysis["sentiment_hq"] = analysis["sentiment_hq"].result()
        analysis["keywords"] = analysis["keywords"].result()
        analysis["ner"] = analysis["ner"].result()

        for i in xrange(len(documents)):
            # Title Analysis
            documents[i]["indico"] = {}
            documents[i]["indico"]["title_sentiment"] = analysis.get('title_sentiment_hq')[i]
            documents[i]["indico"]["title_keywords"] = analysis.get('title_keywords')[i]
            documents[i]["indico"]["title_people"] = analysis.get('title_ner')[i].get("people")
            documents[i]["indico"]["title_places"] = analysis.get('title_ner')[i].get("places")
            documents[i]["indico"]["title_organizations"] = analysis.get('title_ner')[i].get("organizations")

            # Main Text
            documents[i]["indico"]["sentiment"] = analysis.get('sentiment_hq')[i]
            documents[i]["indico"]["keywords"] = analysis.get('keywords')[i]
            documents[i]["indico"]["people"] = analysis.get('ner')[i].get("people")
            documents[i]["indico"]["places"] = analysis.get('ner')[i].get("places")
            documents[i]["indico"]["organizations"] = analysis.get('ner')[i].get("organizations")

            # Summary
            documents[i]["summary"] = summaries[i].result()

        return documents
    except:
        import traceback; traceback.print_exc()
        length = len(documents)/2
        if length <= 1:
            return []
        return add_indico(documents[:length]) + add_indico(documents[length:])

def get_all_data_files(current_dir):
    all_files = []
    for subdir, dirs, files in os.walk(current_dir):
        for filename in files:
            all_files.append(os.path.join(current_dir, filename))
    return all_files

def read_data_file(data_file):
    try:
        lines = [line for line in pe.get_records(file_name=data_file, streaming=True) if len(line.get("description_text", "")) > DESCRIPTION_THRESHOLD]
        root.debug("Parsing Documents for {0}".format(data_file))
        documents = map(lambda x: parse_obj_to_document(x), lines)
        return documents
    except:
        import traceback; traceback.print_exc()
        return []

def upload_data(es, data_file):
    root.debug("Beginning Processing for {0}".format(data_file))
    all_documents = read_data_file(data_file)
    root.debug("Read Data for {0}".format(data_file))
    for documents in partition_all(20, all_documents):
        root.debug("Adding Indico for {0}".format(data_file))
        documents = add_indico(documents)
        root.debug("Uploading to elasticsearch for {0}".format(data_file))
        es.upload(documents)

if __name__ == "__main__":
    executor = ThreadPoolExecutor(max_workers=4)
    es = ESConnection("localhost:9200")
    directory = os.path.join(os.path.dirname(__file__), '../../inputxl')
    files = get_all_data_files(directory)

    futures = []
    for _file in files:
        futures.append(executor.submit(upload_data, es, _file))
    map(lambda x: x.result(), futures)
<|endoftext|>


<commit_before># vim: set ts=4 sw=4 et: coding=UTF-8

import re

from .rpmregexp import RegexpSingle

class Section(object):
    """
    Basic object for parsing each section of spec file.
    It stores the lines in a list and remembers content of
    previous line at hand.
    The unbrace_keywords content is passed from creating
    object to reduce calculation price.
    Various functions do replacement of common typos to
    unificate all the content.
    """

    def __init__(self, specfile):
        self.lines = []
        self.previous_line = None
        self.spec = specfile
        self.reg = RegexpSingle(specfile)
        # Are we inside of conditional or not
        self.condition = False

    def _complete_cleanup(self, line):
        """
        Function that just calls all the cleanups in proper order so it
        can be called beforehand if we override add phase and want to
        do our replaces after cleanup.
        """

        line = line.rstrip()

        if not line.startswith('#'):
            line = self.embrace_macros(line)
            line = self.replace_buildroot(line)
            line = self.replace_optflags(line)
            line = self.replace_known_dirs(line)
            line = self.replace_utils(line)
            line = self.replace_buildservice(line)
            line = self.replace_preamble_macros(line)

        return line

    def _check_conditions(self, line):
        """
        Check if we are in condition that is contained or not
        """
        if self.reg.re_if.match(line):
            self.condition = True
        if self.reg.re_endif.match(line):
            self.condition = False

    def add(self, line):
        """
        Run the cleanup and add the line to the list of lines
        """

        line = self._complete_cleanup(line)

        # condtions detect
        self._check_conditions(line)

        # append to the file
        self.lines.append(line)
        self.previous_line = line

    def output(self, fout, newline = True):
        # Always append one empty line at the end if it is not present
        # and changelog is trailing part of our spec so do not put nothing
        # bellow
        # Also if we are jumping away just after writing one macroed line
        # we don't want to create new line
        if len(self.lines) >= 1:
            if self.lines[-1] != '' and \
                  self.lines[-1] != '%changelog' and not \
                  self.lines[-1].startswith('%if') and \
                  newline:
                self.lines.append('')
            # remove the newlines around ifs if they are not wanted
            if self.lines[-1] == '' and \
                  (self.lines[-2].startswith('%if') or
                  self.lines[-2].startswith('%else')):
                self.lines.pop()

        for line in self.lines:
            fout.write(line + '\n')

    def strip_useless_spaces(self, line):
        """
        Function to remove useless multiple spaces in some areas.
        It can't be called everywhere so we have to call it in
        children classes where fit.
        """
        return ' '.join(line.split())

    def embrace_macros(self, line):
        """
        Add {} around known macros that have no arguments and are not
        on whitelist.
        Whitelist is passed from caller object
        """

        # I don't think that this can be done within one regexp replacement
        # if you have idea, send me a patch :)

        # work only with non-commented part
        sp = line.split('#')
        # so, for now, put braces around everything, what looks like macro,
        previous = sp[0]

        while True:
            sp[0] = self.reg.re_macro.sub(r'\1%{\3}\4', sp[0])
            if sp[0] == previous:
                break
            else:
                previous = sp[0]

        # and replace back known keywords to braceless state again
        sp[0] = self.reg.re_unbrace_keywords.sub(r'%\1', sp[0])
        # re-create the line back
        return '#'.join(sp)

    def replace_buildroot(self, line):
        """
        Replace RPM_BUILD_ROOT for buildroot
        Replace few hard written dirs for further processing with their macro names.
        """
        # FIXME: use regexp to prevent $RPM_BUILD_ROOT_REPLACEMENT
        line = line.replace('${RPM_BUILD_ROOT}', '%{buildroot}')
        line = line.replace('$RPM_BUILD_ROOT', '%{buildroot}')
        line = line.replace('"%{buildroot}"', '%{buildroot}')
        return line

    def replace_optflags(self, line):
        """
        Replace RPM_OPT_FLAGS for %{optflags}
        """
        line = line.replace('${RPM_OPT_FLAGS}', '%{optflags}')
        line = line.replace('$RPM_OPT_FLAGS', '%{optflags}')
        return line

    def replace_known_dirs(self, line):
        """
        Replace hardcoded stuff like /usr/share -> %{_datadir}
        """
        re_bindir = re.compile('%{_prefix}/bin([/\s$])')
        re_sbindir = re.compile('%{_prefix}/sbin([/\s$])')
        re_libexecdir = re.compile('%{_prefix}/lib([/\s$])')
        re_includedir = re.compile('%{_prefix}/include([/\s$])')
        re_datadir = re.compile('%{_prefix}/share([/\s$])')
        re_mandir = re.compile('%{_datadir}/man([/\s$])')
        re_infodir = re.compile('%{_datadir}/info([/\s$])')
        re_docdir = re.compile('%{_datadir}/doc/packages([/\s$])')

        # few hardcoded things
        line = line.replace('/etc/init.d/', '%{_initddir}/')
        line = line.replace('/etc/', '%{_sysconfdir}/')
        line = line.replace('/usr/%{_lib}', '%{_libdir}')
        line = line.replace('/usr/', '%{_prefix}/')
        line = line.replace('/var/', '%{_localstatedir}/')
        line = line.replace('%{_usr}', '%{_prefix}')
        line = line.replace('%{_prefix}/%{_lib}', '%{_libdir}')
        # old typo in rpm macro
        line = line.replace('%_initrddir', '%{_initddir}')
        line = line.replace('%{_initrddir}', '%{_initddir}')

        line = re_bindir.sub(r'%{_bindir}\1', line)
        line = re_sbindir.sub(r'%{_sbindir}\1', line)
        line = re_libexecdir.sub(r'%{_libexecdir}\1', line)
        line = re_includedir.sub(r'%{_includedir}\1', line)
        line = re_datadir.sub(r'%{_datadir}\1', line)
        line = re_mandir.sub(r'%{_mandir}\1', line)
        line = re_infodir.sub(r'%{_infodir}\1', line)
        line = re_docdir.sub(r'%{_docdir}\1', line)

        return line

    def replace_utils(self, line):
        """
        Remove the macro calls for utilities and rather use direct commands.
        OBS ensures there is only one anyway.
        """
        r = {'id_u': 'id -u', 'ln_s': 'ln -s', 'lzma': 'xz --format-lzma', 'mkdir_p': 'mkdir -p', 'awk':'gawk', 'cc':'gcc', 'cpp':'gcc -E', 'cxx':'g++', 'remsh':'rsh', }
        for i in r:
            line = line.replace('%{__' + i + '}', r[i])

        for i in [ 'aclocal', 'ar', 'as', 'autoconf', 'autoheader', 'automake', 'bzip2', 'cat', 'chgrp', 'chmod', 'chown', 'cp', 'cpio', 'file', 'gpg', 'grep', 'gzip', 'id', 'install', 'ld', 'libtoolize', 'make', 'mkdir', 'mv', 'nm', 'objcopy', 'objdump', 'patch', 'perl', 'python', 'ranlib', 'restorecon', 'rm', 'rsh', 'sed', 'semodule', 'ssh', 'strip', 'tar', 'unzip', 'xz', ]:
            line = line.replace('%{__' + i + '}', i)

        return line

    def replace_buildservice(self, line):
        """
        Pretty format the conditions for distribution/version detection.
        Replace %{suse_version} for 0%{?suse_version}
        """
        for i in ['centos', 'debian', 'fedora', 'mandriva', 'meego', 'rhel', 'sles', 'suse', 'ubuntu']:
            line = line.replace('%{' + i + '_version}', '0%{?' + i + '_version}').replace('00%{','0%{')
        return line

    def replace_preamble_macros(self, line):
        """
        Replace %{S:0} for %{SOURCE0} and so on.
        """
        line = re.sub(r'%{P:(\d+)}', r'%{PATCH\1}', line)
        line = re.sub(r'%{S:(\d+)}', r'%{SOURCE\1}', line)

        return line
<commit_msg>Add array size check to avoid out-of-range crash.<commit_after># vim: set ts=4 sw=4 et: coding=UTF-8

import re

from .rpmregexp import RegexpSingle

class Section(object):
    """
    Basic object for parsing each section of spec file.
    It stores the lines in a list and remembers content of
    previous line at hand.
    The unbrace_keywords content is passed from creating
    object to reduce calculation price.
    Various functions do replacement of common typos to
    unificate all the content.
    """

    def __init__(self, specfile):
        self.lines = []
        self.previous_line = None
        self.spec = specfile
        self.reg = RegexpSingle(specfile)
        # Are we inside of conditional or not
        self.condition = False

    def _complete_cleanup(self, line):
        """
        Function that just calls all the cleanups in proper order so it
        can be called beforehand if we override add phase and want to
        do our replaces after cleanup.
        """

        line = line.rstrip()

        if not line.startswith('#'):
            line = self.embrace_macros(line)
            line = self.replace_buildroot(line)
            line = self.replace_optflags(line)
            line = self.replace_known_dirs(line)
            line = self.replace_utils(line)
            line = self.replace_buildservice(line)
            line = self.replace_preamble_macros(line)

        return line

    def _check_conditions(self, line):
        """
        Check if we are in condition that is contained or not
        """
        if self.reg.re_if.match(line):
            self.condition = True
        if self.reg.re_endif.match(line):
            self.condition = False

    def add(self, line):
        """
        Run the cleanup and add the line to the list of lines
        """

        line = self._complete_cleanup(line)

        # condtions detect
        self._check_conditions(line)

        # append to the file
        self.lines.append(line)
        self.previous_line = line

    def output(self, fout, newline = True):
        # Always append one empty line at the end if it is not present
        # and changelog is trailing part of our spec so do not put nothing
        # bellow
        # Also if we are jumping away just after writing one macroed line
        # we don't want to create new line
        if len(self.lines) >= 1:
            if self.lines[-1] != '' and \
                  self.lines[-1] != '%changelog' and not \
                  self.lines[-1].startswith('%if') and \
                  newline:
                self.lines.append('')
            # remove the newlines around ifs if they are not wanted
            if len(self.lines) >= 2:
                if self.lines[-1] == '' and \
                      (self.lines[-2].startswith('%if') or
                      self.lines[-2].startswith('%else')):
                    self.lines.pop()

        for line in self.lines:
            fout.write(line + '\n')

    def strip_useless_spaces(self, line):
        """
        Function to remove useless multiple spaces in some areas.
        It can't be called everywhere so we have to call it in
        children classes where fit.
        """
        return ' '.join(line.split())

    def embrace_macros(self, line):
        """
        Add {} around known macros that have no arguments and are not
        on whitelist.
        Whitelist is passed from caller object
        """

        # I don't think that this can be done within one regexp replacement
        # if you have idea, send me a patch :)

        # work only with non-commented part
        sp = line.split('#')
        # so, for now, put braces around everything, what looks like macro,
        previous = sp[0]

        while True:
            sp[0] = self.reg.re_macro.sub(r'\1%{\3}\4', sp[0])
            if sp[0] == previous:
                break
            else:
                previous = sp[0]

        # and replace back known keywords to braceless state again
        sp[0] = self.reg.re_unbrace_keywords.sub(r'%\1', sp[0])
        # re-create the line back
        return '#'.join(sp)

    def replace_buildroot(self, line):
        """
        Replace RPM_BUILD_ROOT for buildroot
        Replace few hard written dirs for further processing with their macro names.
        """
        # FIXME: use regexp to prevent $RPM_BUILD_ROOT_REPLACEMENT
        line = line.replace('${RPM_BUILD_ROOT}', '%{buildroot}')
        line = line.replace('$RPM_BUILD_ROOT', '%{buildroot}')
        line = line.replace('"%{buildroot}"', '%{buildroot}')
        return line

    def replace_optflags(self, line):
        """
        Replace RPM_OPT_FLAGS for %{optflags}
        """
        line = line.replace('${RPM_OPT_FLAGS}', '%{optflags}')
        line = line.replace('$RPM_OPT_FLAGS', '%{optflags}')
        return line

    def replace_known_dirs(self, line):
        """
        Replace hardcoded stuff like /usr/share -> %{_datadir}
        """
        re_bindir = re.compile('%{_prefix}/bin([/\s$])')
        re_sbindir = re.compile('%{_prefix}/sbin([/\s$])')
        re_libexecdir = re.compile('%{_prefix}/lib([/\s$])')
        re_includedir = re.compile('%{_prefix}/include([/\s$])')
        re_datadir = re.compile('%{_prefix}/share([/\s$])')
        re_mandir = re.compile('%{_datadir}/man([/\s$])')
        re_infodir = re.compile('%{_datadir}/info([/\s$])')
        re_docdir = re.compile('%{_datadir}/doc/packages([/\s$])')

        # few hardcoded things
        line = line.replace('/etc/init.d/', '%{_initddir}/')
        line = line.replace('/etc/', '%{_sysconfdir}/')
        line = line.replace('/usr/%{_lib}', '%{_libdir}')
        line = line.replace('/usr/', '%{_prefix}/')
        line = line.replace('/var/', '%{_localstatedir}/')
        line = line.replace('%{_usr}', '%{_prefix}')
        line = line.replace('%{_prefix}/%{_lib}', '%{_libdir}')
        # old typo in rpm macro
        line = line.replace('%_initrddir', '%{_initddir}')
        line = line.replace('%{_initrddir}', '%{_initddir}')

        line = re_bindir.sub(r'%{_bindir}\1', line)
        line = re_sbindir.sub(r'%{_sbindir}\1', line)
        line = re_libexecdir.sub(r'%{_libexecdir}\1', line)
        line = re_includedir.sub(r'%{_includedir}\1', line)
        line = re_datadir.sub(r'%{_datadir}\1', line)
        line = re_mandir.sub(r'%{_mandir}\1', line)
        line = re_infodir.sub(r'%{_infodir}\1', line)
        line = re_docdir.sub(r'%{_docdir}\1', line)

        return line

    def replace_utils(self, line):
        """
        Remove the macro calls for utilities and rather use direct commands.
        OBS ensures there is only one anyway.
        """
        r = {'id_u': 'id -u', 'ln_s': 'ln -s', 'lzma': 'xz --format-lzma', 'mkdir_p': 'mkdir -p', 'awk':'gawk', 'cc':'gcc', 'cpp':'gcc -E', 'cxx':'g++', 'remsh':'rsh', }
        for i in r:
            line = line.replace('%{__' + i + '}', r[i])

        for i in [ 'aclocal', 'ar', 'as', 'autoconf', 'autoheader', 'automake', 'bzip2', 'cat', 'chgrp', 'chmod', 'chown', 'cp', 'cpio', 'file', 'gpg', 'grep', 'gzip', 'id', 'install', 'ld', 'libtoolize', 'make', 'mkdir', 'mv', 'nm', 'objcopy', 'objdump', 'patch', 'perl', 'python', 'ranlib', 'restorecon', 'rm', 'rsh', 'sed', 'semodule', 'ssh', 'strip', 'tar', 'unzip', 'xz', ]:
            line = line.replace('%{__' + i + '}', i)

        return line

    def replace_buildservice(self, line):
        """
        Pretty format the conditions for distribution/version detection.
        Replace %{suse_version} for 0%{?suse_version}
        """
        for i in ['centos', 'debian', 'fedora', 'mandriva', 'meego', 'rhel', 'sles', 'suse', 'ubuntu']:
            line = line.replace('%{' + i + '_version}', '0%{?' + i + '_version}').replace('00%{','0%{')
        return line

    def replace_preamble_macros(self, line):
        """
        Replace %{S:0} for %{SOURCE0} and so on.
        """
        line = re.sub(r'%{P:(\d+)}', r'%{PATCH\1}', line)
        line = re.sub(r'%{S:(\d+)}', r'%{SOURCE\1}', line)
        return line
<|endoftext|>


<commit_before>"""Integration with nest devices."""

import json
import logging
import urllib
import urllib2

from google.appengine.ext import ndb

from appengine import account, device, rest, user
from common import creds

@account.register('netatmo')
class NetatmoAccount(account.Account):
  """Class represents a Netatmo account."""
  AUTH_URL = ('https://api.netatmo.net/oauth2/authorize?'
              'client_id=%(client_id)s&state=%(state)s')
  ACCESS_TOKEN_URL = 'https://api.netatmo.net/oauth2/token'
  SCOPES = 'read_station read_thermostat write_thermostat'

  CLIENT_ID = creds.NETATMO_CLIENT_ID
  CLIENT_SECRET = creds.NETATMO_CLIENT_SECRET

  def _get_refresh_data(self):
    return urllib.urlencode({'grant_type': 'authorization_code',
                             'client_id': self.CLIENT_ID,
                             'client_secret': self.CLIENT_SECRET,
                             'code': self.auth_code,
                             'redirect_uri': account.REDIRECT_URL,
                             'scope': self.SCOPES})

  @rest.command
  def refresh_devices(self):
    if self.access_token is None:
      logging.info('No access token, skipping.')
      return

<commit_msg>Query netatmo api periodically.<commit_after>"""Integration with nest devices."""

import json
import logging
import urllib
import urllib2

from google.appengine.ext import ndb

from appengine import account, device, rest, user
from common import creds

@account.register('netatmo')
class NetatmoAccount(account.Account):
  """Class represents a Netatmo account."""
  AUTH_URL = ('https://api.netatmo.net/oauth2/authorize?'
              'client_id=%(client_id)s&state=%(state)s')
  ACCESS_TOKEN_URL = 'https://api.netatmo.net/oauth2/token'
  SCOPES = 'read_station read_thermostat write_thermostat'
  API_URL = ('https://api.netatmo.net/api/devicelist?'
             'access_token=%(access_token)s')

  CLIENT_ID = creds.NETATMO_CLIENT_ID
  CLIENT_SECRET = creds.NETATMO_CLIENT_SECRET

  def _get_refresh_data(self):
    return urllib.urlencode({'grant_type': 'authorization_code',
                             'client_id': self.CLIENT_ID,
                             'client_secret': self.CLIENT_SECRET,
                             'code': self.auth_code,
                             'redirect_uri': account.REDIRECT_URL,
                             'scope': self.SCOPES})

  @rest.command
  def refresh_devices(self):
    if self.access_token is None:
      logging.info('No access token, skipping.')
      return

    url = self.API_URL % {'access_token': self.access_token}
    result = urllib2.urlopen(url)
    result = json.load(result)
    logging.info(result)

<|endoftext|>


<commit_before>"""This module contains the forms for the members package

.. module:: application.members.forms

.. moduleauthor:: Devin Schwab <dts34@case.edu>
"""

from flaskext import wtf
from flaskext.wtf import validators

class RoleForm(wtf.Form):
    name = wtf.TextField('Role Name', [validators.Required()])

class AddressForm(wtf.Form):
    addrName = wtf.TextField('Name', [validators.Optional()])
    street1 = wtf.TextField('Street', [validators.Required()])
    street2 = wtf.TextField('', [validators.Optional()])
    city = wtf.TextField('City', [validators.Required()])
    state = wtf.TextField('State', [validators.Required()])
    zip = wtf.IntegerField('Zip', [validators.Required()])

class PhoneNumberForm(wtf.Form):
    phoneName = wtf.TextField('Name', [validators.Optional()])
    phoneNumber = wtf.TextField('Phone Number', [validators.Required()])

class EmailAddressForm(wtf.Form):
    emailName = wtf.TextField('Name', [validators.Optional()])
    emailAddress = wtf.TextField('Email', [validators.Email()])

class SearchUserForm(wtf.Form):
    fName = wtf.TextField('First Name', [validators.Optional()])
    mName = wtf.TextField('Middle Name', [validators.Optional()])
    lName = wtf.TextField('Last Name', [validators.Optional()])

class CreateUserForm(wtf.Form):
    fName = wtf.TextField('First Name', [validators.Required()])
    mName = wtf.TextField('Middle Name', [validators.Optional()])
    lName = wtf.TextField('Last Name', [validators.Required()])
    cwruid = wtf.TextField('CWRU ID', [validators.Required()])
    family = wtf.TextField('Family Name', [validators.Optional()])
    big = wtf.TextField('Big CWRU ID', [validators.Optional()])
    avatar = wtf.TextField('Gravatar email', [validators.Email()])
    roles = wtf.FieldList(wtf.SelectMultipleField('Role Name', [validators.Optional()]))

<commit_msg>Removed the unecessary RoleForm<commit_after>"""This module contains the forms for the members package

.. module:: application.members.forms

.. moduleauthor:: Devin Schwab <dts34@case.edu>
"""

from flaskext import wtf
from flaskext.wtf import validators

class AddressForm(wtf.Form):
    addrName = wtf.TextField('Name', [validators.Optional()])
    street1 = wtf.TextField('Street', [validators.Required()])
    street2 = wtf.TextField('', [validators.Optional()])
    city = wtf.TextField('City', [validators.Required()])
    state = wtf.TextField('State', [validators.Required()])
    zip = wtf.IntegerField('Zip', [validators.Required()])

class PhoneNumberForm(wtf.Form):
    phoneName = wtf.TextField('Name', [validators.Optional()])
    phoneNumber = wtf.TextField('Phone Number', [validators.Required()])

class EmailAddressForm(wtf.Form):
    emailName = wtf.TextField('Name', [validators.Optional()])
    emailAddress = wtf.TextField('Email', [validators.Email()])

class SearchUserForm(wtf.Form):
    fName = wtf.TextField('First Name', [validators.Optional()])
    mName = wtf.TextField('Middle Name', [validators.Optional()])
    lName = wtf.TextField('Last Name', [validators.Optional()])

class CreateUserForm(wtf.Form):
    fName = wtf.TextField('First Name', [validators.Required()])
    mName = wtf.TextField('Middle Name', [validators.Optional()])
    lName = wtf.TextField('Last Name', [validators.Required()])
    cwruid = wtf.TextField('CWRU ID', [validators.Required()])
    family = wtf.TextField('Family Name', [validators.Optional()])
    big = wtf.TextField('Big CWRU ID', [validators.Optional()])
    avatar = wtf.TextField('Gravatar email', [validators.Email()])
    roles = wtf.FieldList(wtf.SelectMultipleField('Role Name', [validators.Optional()]))

<|endoftext|>


<commit_before>"""Test aspects of lldb commands on universal binaries."""

import os, time
import unittest2
import lldb
from lldbtest import *

class UniversalTestCase(TestBase):

    mydir = os.path.join("macosx", "universal")

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line number to break inside main().
        self.line = line_number('main.c', '// Set break point at this line.')

    # rdar://problem/8972204 AddressByteSize of 32-bit process should be 4, got 8 instead.
    @unittest2.skipUnless(sys.platform.startswith("darwin") and os.uname()[4]=='i386',
                          "requires Darwin & i386")
    def test_process_launch_for_universal(self):
        """Test process launch of a universal binary."""
        from lldbutil import print_registers

        # Invoke the default build rule.
        self.buildDefault()

        # Note that "testit" is a universal binary.
        exe = os.path.join(os.getcwd(), "testit")

        # By default, x86_64 is assumed if no architecture is specified.
        self.expect("file " + exe, CURRENT_EXECUTABLE_SET,
            startstr = "Current executable set to ",
            substrs = ["testit' (x86_64)."])

        # Break inside the main.
        self.expect("breakpoint set -f main.c -l %d" % self.line,
                    BREAKPOINT_CREATED,
            startstr = "Breakpoint created: 1: file ='main.c', line = %d, locations = 1" %
                        self.line)

        # We should be able to launch the x86_64 executable.
        self.runCmd("run", RUN_SUCCEEDED)

        # Check whether we have a 64-bit process launched.
        target = self.dbg.GetSelectedTarget()
        process = target.GetProcess()
        self.assertTrue(target and process and
                        self.invoke(process, 'GetAddressByteSize') == 8,
                        "64-bit process launched")

        frame = process.GetThreadAtIndex(0).GetFrameAtIndex(0)
        registers = print_registers(frame, string_buffer=True)
        self.expect(registers, exe=False,
            substrs = ['Name: rax'])

        self.runCmd("continue")

        # Now specify i386 as the architecture for "testit".
        self.expect("file " + exe + " -a i386", CURRENT_EXECUTABLE_SET,
            startstr = "Current executable set to ",
            substrs = ["testit' (i386)."])

        # Break inside the main.
        self.expect("breakpoint set -f main.c -l %d" % self.line,
                    BREAKPOINT_CREATED,
            startstr = "Breakpoint created: 1: file ='main.c', line = %d, locations = 1" %
                        self.line)

        # We should be able to launch the i386 executable as well.
        self.runCmd("run", RUN_SUCCEEDED)

        # Check whether we have a 32-bit process launched.
        target = self.dbg.GetSelectedTarget()
        process = target.GetProcess()
        self.assertTrue(target and process,
                        "32-bit process launched")

        pointerSize = self.invoke(process, 'GetAddressByteSize')
        self.assertTrue(pointerSize == 4,
                        "AddressByteSize of 32-bit process should be 4, got %d instead." % pointerSize)

        frame = process.GetThreadAtIndex(0).GetFrameAtIndex(0)
        registers = print_registers(frame, string_buffer=True)
        self.expect(registers, exe=False,
            substrs = ['Name: eax'])

        self.runCmd("continue")

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<commit_msg>Modify the skipUnless() logic to work for OSX Lion.<commit_after>"""Test aspects of lldb commands on universal binaries."""

import os, time
import unittest2
import lldb
from lldbtest import *

class UniversalTestCase(TestBase):

    mydir = os.path.join("macosx", "universal")

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line number to break inside main().
        self.line = line_number('main.c', '// Set break point at this line.')

    # rdar://problem/8972204 AddressByteSize of 32-bit process should be 4, got 8 instead.
    @unittest2.skipUnless(sys.platform.startswith("darwin") and os.uname()[4] in ['i386', 'x86_64'],
                          "requires Darwin & i386")
    def test_process_launch_for_universal(self):
        """Test process launch of a universal binary."""
        from lldbutil import print_registers

        # Invoke the default build rule.
        self.buildDefault()

        # Note that "testit" is a universal binary.
        exe = os.path.join(os.getcwd(), "testit")

        # By default, x86_64 is assumed if no architecture is specified.
        self.expect("file " + exe, CURRENT_EXECUTABLE_SET,
            startstr = "Current executable set to ",
            substrs = ["testit' (x86_64)."])

        # Break inside the main.
        self.expect("breakpoint set -f main.c -l %d" % self.line,
                    BREAKPOINT_CREATED,
            startstr = "Breakpoint created: 1: file ='main.c', line = %d, locations = 1" %
                        self.line)

        # We should be able to launch the x86_64 executable.
        self.runCmd("run", RUN_SUCCEEDED)

        # Check whether we have a 64-bit process launched.
        target = self.dbg.GetSelectedTarget()
        process = target.GetProcess()
        self.assertTrue(target and process and
                        self.invoke(process, 'GetAddressByteSize') == 8,
                        "64-bit process launched")

        frame = process.GetThreadAtIndex(0).GetFrameAtIndex(0)
        registers = print_registers(frame, string_buffer=True)
        self.expect(registers, exe=False,
            substrs = ['Name: rax'])

        self.runCmd("continue")

        # Now specify i386 as the architecture for "testit".
        self.expect("file " + exe + " -a i386", CURRENT_EXECUTABLE_SET,
            startstr = "Current executable set to ",
            substrs = ["testit' (i386)."])

        # Break inside the main.
        self.expect("breakpoint set -f main.c -l %d" % self.line,
                    BREAKPOINT_CREATED,
            startstr = "Breakpoint created: 1: file ='main.c', line = %d, locations = 1" %
                        self.line)

        # We should be able to launch the i386 executable as well.
        self.runCmd("run", RUN_SUCCEEDED)

        # Check whether we have a 32-bit process launched.
        target = self.dbg.GetSelectedTarget()
        process = target.GetProcess()
        self.assertTrue(target and process,
                        "32-bit process launched")

        pointerSize = self.invoke(process, 'GetAddressByteSize')
        self.assertTrue(pointerSize == 4,
                        "AddressByteSize of 32-bit process should be 4, got %d instead." % pointerSize)

        frame = process.GetThreadAtIndex(0).GetFrameAtIndex(0)
        registers = print_registers(frame, string_buffer=True)
        self.expect(registers, exe=False,
            substrs = ['Name: eax'])

        self.runCmd("continue")

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<|endoftext|>


<commit_before>#!/usr/bin/python3

# BuildFox ninja generator

import os
import re
import copy
import argparse

from lib_engine import Engine
from lib_environment import discover

# core definitions -----------------------------------------------------------

fox_core = r"""
# buildfox core configuration

# buildfox relies on scoped rules and they were added in ninja v1.6
ninja_required_version = 1.6

filter toolset:msvc
	# msvc support
	cxx = cl
	lib = lib

	rule cxx
		command = $cxx $cxxflags $defines $includedirs $disable_warnings /nologo /showIncludes -c $in /Fo$out
		description = cxx $in
		deps = msvc
		expand = true

	rule link
		command = $cxx /nologo @$out.rsp /link $ldflags $libdirs $ignore_default_libs /out:$out
		description = link $out
		rspfile = $out.rsp
		rspfile_content = $in

	rule link_dll
		command = $cxx /nologo @$out.rsp /link /DLL $ldflags $libdirs $ignore_default_libs /out:$out
		description = link $out
		rspfile = $out.rsp
		rspfile_content = $in

	rule lib
		command = $lib $libflags @$out.rsp /nologo -OUT:$out
		description = lib $out
		rspfile = $out.rsp
		rspfile_content = $in

	auto r"(?i).*\.obj": cxx r"(?i).*\.(cpp|cxx|cc|c\+\+|c)$"
	auto r"(?i).*\.exe": link r"(?i).*\.(obj|lib)$"
	auto r"(?i).*\.dll": link_dll r"(?i).*\.(obj|lib)$"
	auto r"(?i).*\.lib": lib r"(?i).*\.(obj|lib)$"

	# extensions transformers
	transformer app: ${param}.exe
	transformer obj: ${param}.obj
	transformer lib: ${param}.lib
	transformer shlib: ${param}.dll

	# MSVC flags
	# more info here https://msdn.microsoft.com/en-us/library/19z1t1wy.aspx

	# optimizations
	cxx_omit_frame_pointer = /Oy
	cxx_disable_optimizations = /Od
	cxx_full_optimizations = /Ox
	cxx_size_optimizations = /O1
	cxx_speed_optimizations = /O2

	# code generation
	cxx_exceptions = /EHsc
	cxx_no_exceptions = /EHsc- # TODO not sure about this one
	cxx_seh_exceptions = /EHa
	cxx_whole_program_optimizations = /GL
	cxx_rtti = /GR
	cxx_no_rtti = /GR-
	cxx_clr = /clr
	cxx_clr_pure = /clr:pure
	cxx_clr_safe = /clr:safe
	cxx_multithread_compilation = /MP
	cxx_mimimal_rebuild = /Gm
	cxx_no_mimimal_rebuild = /Gm-
	cxx_floatpoint_fast = /fp:fast
	cxx_floatpoint_strict = /fp:strict
	cxx_cdecl = /Gd
	cxx_fastcall = /Gr
	cxx_stdcall = /Gz
	cxx_vectorcall = /Gv
	cxx_avx = /arch:AVX
	cxx_avx2 = /arch:AVX2
	cxx_sse = /arch:SSE
	cxx_sse2 = /arch:SSE2

	# language
	cxx_symbols = /Z7
	cxx_omit_default_lib = /Zl

	# linking
	cxx_runtime_static_debug = /MTd
	cxx_runtime_dynamic_debug = /MDd
	cxx_runtime_static_release = /MT
	cxx_runtime_dynamic_release = /MD

	# miscellaneous
	cxx_fatal_warnings = /WX
	cxx_extra_warnings = /W4
	cxx_no_warnings = /W0

	# linker flags
	ld_no_incremental_link = /INCREMENTAL:NO
	ld_no_manifest = /MANIFEST:NO
	ld_ignore_default_libs = /NODEFAULTLIB
	ld_symbols = /DEBUG
	ld_shared_lib = /DLL

	# transformers
	defines =
	includedirs =
	disable_warnings =
	libdirs =
	ignore_default_libs =
	transformer defines: /D${param}
	transformer includedirs: /I${param}
	transformer disable_warnings: /wd${param}
	transformer libdirs: /LIBPATH:${param}
	transformer ignore_default_libs: /NODEFAULTLIB:${param}

	# main flags
	cxxflags =
	ldflags =
	libflags =
	filter variation:debug
		cxxflags = $cxx_disable_optimizations $cxx_symbols
		ldflags = $ld_symbols
	filter variation:release
		cxxflags = $cxx_speed_optimizations
		ldflags =

filter toolset:clang
	# clang suport
	rule cxx
		command = clang++ -c $in -o $out -MMD $cxxflags $defines $includedirs
		description = cxx $in
		depfile = $out.d
		deps = gcc
		expand = true

	rule link
		command = clang++ $in -o $out $ldflags $libdirs
		description = link $out

	auto r"(?i).*\.o": cxx r"(?i).*\.(cpp|cxx|cc|c\+\+)$"

	# Clang flags
	# more info here http://clang.llvm.org/docs/CommandGuide/clang.html
	# TODO:

	# optimizations
	cxx_omit_frame_pointer = -fomit-frame-pointer
	cxx_disable_optimizations = -O0
	cxx_full_optimizations = -O3
	cxx_size_optimizations = -Os
	cxx_speed_optimizations = -Ofast

	# code generation
	cxx_exceptions = -fexceptions
	cxx_no_exceptions = -fno-exceptions
	cxx_whole_program_optimizations = -O4
	cxx_rtti = -frtti
	cxx_no_rtti = -fno-rtti
	cxx_floatpoint_fast = -funsafe-math-optimizations
	cxx_avx = -mavx
	cxx_avx2 = -mavx2
	cxx_sse = -msse
	cxx_sse2 = -msse2

	# miscellaneous
	cxx_fatal_warnings = -Werror
	cxx_extra_warnings = -Wall -Wextra
	cxx_no_warnings = -w

	# transformers
	defines =
	includedirs =
	libdirs =
	transformer defines: -D${param}
	transformer includedirs: -I${param}
	transformer libdirs: -L${param}

	# main flags
	cxxflags =
	ldflags =
	filter variation:debug
		cxxflags = -g
		ldflags = -g
"""

# main app -----------------------------------------------------------

argsparser = argparse.ArgumentParser(description = "buildfox ninja generator")
argsparser.add_argument("-i", "--in", help = "input file", default = "build.fox")
argsparser.add_argument("-o", "--out", help = "output file", default = "build.ninja")
argsparser.add_argument("-w", "--workdir", help = "working directory")
argsparser.add_argument("variables", metavar = "name=value", type = str, nargs = "*", help = "variables with values to setup", default = [])
#argsparser.add_argument("-v", "--verbose", action = "store_true", help = "verbose output") # TODO
argsparser.add_argument("--no-core", action = "store_false",
	help = "disable parsing fox core definitions", default = True, dest = "core")
argsparser.add_argument("--no-env", action = "store_false",
	help = "disable environment discovery", default = True, dest = "env")
args = vars(argsparser.parse_args())

if args.get("workdir"):
	os.chdir(args.get("workdir"))

engine = Engine()

if args.get("env"):
	vars = discover()
	for name in sorted(vars.keys()):
		engine.on_assign((name, vars.get(name), "="))

for var in args.get("variables"):
	parts = var.split("=")
	if len(parts) == 2:
		name, value = parts[0], parts[1]
		engine.on_assign((name, value, "="))
	else:
		raise SyntaxError("unknown argument '%s'. you should use name=value syntax to setup a variable" % var)

if args.get("core"):
	engine.load_core(fox_core)

engine.load(args.get("in"))
engine.save(args.get("out"))
<commit_msg>Shebang fixed to use first python from the environment, related to #41.<commit_after>#!/usr/bin/env python

# BuildFox ninja generator

import os
import re
import copy
import argparse

from lib_engine import Engine
from lib_environment import discover

# core definitions -----------------------------------------------------------

fox_core = r"""
# buildfox core configuration

# buildfox relies on scoped rules and they were added in ninja v1.6
ninja_required_version = 1.6

filter toolset:msvc
	# msvc support
	cxx = cl
	lib = lib

	rule cxx
		command = $cxx $cxxflags $defines $includedirs $disable_warnings /nologo /showIncludes -c $in /Fo$out
		description = cxx $in
		deps = msvc
		expand = true

	rule link
		command = $cxx /nologo @$out.rsp /link $ldflags $libdirs $ignore_default_libs /out:$out
		description = link $out
		rspfile = $out.rsp
		rspfile_content = $in

	rule link_dll
		command = $cxx /nologo @$out.rsp /link /DLL $ldflags $libdirs $ignore_default_libs /out:$out
		description = link $out
		rspfile = $out.rsp
		rspfile_content = $in

	rule lib
		command = $lib $libflags @$out.rsp /nologo -OUT:$out
		description = lib $out
		rspfile = $out.rsp
		rspfile_content = $in

	auto r"(?i).*\.obj": cxx r"(?i).*\.(cpp|cxx|cc|c\+\+|c)$"
	auto r"(?i).*\.exe": link r"(?i).*\.(obj|lib)$"
	auto r"(?i).*\.dll": link_dll r"(?i).*\.(obj|lib)$"
	auto r"(?i).*\.lib": lib r"(?i).*\.(obj|lib)$"

	# extensions transformers
	transformer app: ${param}.exe
	transformer obj: ${param}.obj
	transformer lib: ${param}.lib
	transformer shlib: ${param}.dll

	# MSVC flags
	# more info here https://msdn.microsoft.com/en-us/library/19z1t1wy.aspx

	# optimizations
	cxx_omit_frame_pointer = /Oy
	cxx_disable_optimizations = /Od
	cxx_full_optimizations = /Ox
	cxx_size_optimizations = /O1
	cxx_speed_optimizations = /O2

	# code generation
	cxx_exceptions = /EHsc
	cxx_no_exceptions = /EHsc- # TODO not sure about this one
	cxx_seh_exceptions = /EHa
	cxx_whole_program_optimizations = /GL
	cxx_rtti = /GR
	cxx_no_rtti = /GR-
	cxx_clr = /clr
	cxx_clr_pure = /clr:pure
	cxx_clr_safe = /clr:safe
	cxx_multithread_compilation = /MP
	cxx_mimimal_rebuild = /Gm
	cxx_no_mimimal_rebuild = /Gm-
	cxx_floatpoint_fast = /fp:fast
	cxx_floatpoint_strict = /fp:strict
	cxx_cdecl = /Gd
	cxx_fastcall = /Gr
	cxx_stdcall = /Gz
	cxx_vectorcall = /Gv
	cxx_avx = /arch:AVX
	cxx_avx2 = /arch:AVX2
	cxx_sse = /arch:SSE
	cxx_sse2 = /arch:SSE2

	# language
	cxx_symbols = /Z7
	cxx_omit_default_lib = /Zl

	# linking
	cxx_runtime_static_debug = /MTd
	cxx_runtime_dynamic_debug = /MDd
	cxx_runtime_static_release = /MT
	cxx_runtime_dynamic_release = /MD

	# miscellaneous
	cxx_fatal_warnings = /WX
	cxx_extra_warnings = /W4
	cxx_no_warnings = /W0

	# linker flags
	ld_no_incremental_link = /INCREMENTAL:NO
	ld_no_manifest = /MANIFEST:NO
	ld_ignore_default_libs = /NODEFAULTLIB
	ld_symbols = /DEBUG
	ld_shared_lib = /DLL

	# transformers
	defines =
	includedirs =
	disable_warnings =
	libdirs =
	ignore_default_libs =
	transformer defines: /D${param}
	transformer includedirs: /I${param}
	transformer disable_warnings: /wd${param}
	transformer libdirs: /LIBPATH:${param}
	transformer ignore_default_libs: /NODEFAULTLIB:${param}

	# main flags
	cxxflags =
	ldflags =
	libflags =
	filter variation:debug
		cxxflags = $cxx_disable_optimizations $cxx_symbols
		ldflags = $ld_symbols
	filter variation:release
		cxxflags = $cxx_speed_optimizations
		ldflags =

filter toolset:clang
	# clang suport
	rule cxx
		command = clang++ -c $in -o $out -MMD $cxxflags $defines $includedirs
		description = cxx $in
		depfile = $out.d
		deps = gcc
		expand = true

	rule link
		command = clang++ $in -o $out $ldflags $libdirs
		description = link $out

	auto r"(?i).*\.o": cxx r"(?i).*\.(cpp|cxx|cc|c\+\+)$"

	# Clang flags
	# more info here http://clang.llvm.org/docs/CommandGuide/clang.html
	# TODO:

	# optimizations
	cxx_omit_frame_pointer = -fomit-frame-pointer
	cxx_disable_optimizations = -O0
	cxx_full_optimizations = -O3
	cxx_size_optimizations = -Os
	cxx_speed_optimizations = -Ofast

	# code generation
	cxx_exceptions = -fexceptions
	cxx_no_exceptions = -fno-exceptions
	cxx_whole_program_optimizations = -O4
	cxx_rtti = -frtti
	cxx_no_rtti = -fno-rtti
	cxx_floatpoint_fast = -funsafe-math-optimizations
	cxx_avx = -mavx
	cxx_avx2 = -mavx2
	cxx_sse = -msse
	cxx_sse2 = -msse2

	# miscellaneous
	cxx_fatal_warnings = -Werror
	cxx_extra_warnings = -Wall -Wextra
	cxx_no_warnings = -w

	# transformers
	defines =
	includedirs =
	libdirs =
	transformer defines: -D${param}
	transformer includedirs: -I${param}
	transformer libdirs: -L${param}

	# main flags
	cxxflags =
	ldflags =
	filter variation:debug
		cxxflags = -g
		ldflags = -g
"""

# main app -----------------------------------------------------------

argsparser = argparse.ArgumentParser(description = "buildfox ninja generator")
argsparser.add_argument("-i", "--in", help = "input file", default = "build.fox")
argsparser.add_argument("-o", "--out", help = "output file", default = "build.ninja")
argsparser.add_argument("-w", "--workdir", help = "working directory")
argsparser.add_argument("variables", metavar = "name=value", type = str, nargs = "*", help = "variables with values to setup", default = [])
#argsparser.add_argument("-v", "--verbose", action = "store_true", help = "verbose output") # TODO
argsparser.add_argument("--no-core", action = "store_false",
	help = "disable parsing fox core definitions", default = True, dest = "core")
argsparser.add_argument("--no-env", action = "store_false",
	help = "disable environment discovery", default = True, dest = "env")
args = vars(argsparser.parse_args())

if args.get("workdir"):
	os.chdir(args.get("workdir"))

engine = Engine()

if args.get("env"):
	vars = discover()
	for name in sorted(vars.keys()):
		engine.on_assign((name, vars.get(name), "="))

for var in args.get("variables"):
	parts = var.split("=")
	if len(parts) == 2:
		name, value = parts[0], parts[1]
		engine.on_assign((name, value, "="))
	else:
		raise SyntaxError("unknown argument '%s'. you should use name=value syntax to setup a variable" % var)

if args.get("core"):
	engine.load_core(fox_core)

engine.load(args.get("in"))
engine.save(args.get("out"))
<|endoftext|>