<commit_before>import os
from os import path
from urllib.parse import urljoin

from nib import Document, Resource, Render
from nib.processor import preprocessors, postprocessors,\
    document_processors, resource_processors, markup_processors
import nib.plugins

class Build(object):
    def __init__(self, options):
        self.document_path = options['document_path']
        self.resource_path = options['resource_path']
        self.output_path = options['output_path']
        self.options = options

        nib.plugins.load(options)

    def load(self):
        documents = []

        for root, dirs, files in os.walk(self.document_path):
            for filename in files:
                filepath = path.join(root, filename)

                try:
                    print('Reading {}'.format(filepath))
                    document = Document.from_file(filepath, self.options)
                    documents.append(document)

                except Exception as e:
                    print('Error while reading {}: {}'.format(filepath, e))

        resources = []

        for root, dirs, files in os.walk(self.resource_path):
            for filename in files:
                filepath = path.join(root, filename)

                try:
                    print('Reading {}'.format(filepath))
                    resource = Resource.from_file(filepath, self.options)
                    resources.append(resource)

                except Exception as e:
                    print('Error while reading {}: {}'.format(filepath, e))

        return documents, resources

    def process_documents(self, documents, resources):
        # break documents into groups by type
        documents_by_group = {}
        for document in documents:
            group = document.group
            if group not in documents_by_group:
                documents_by_group[group] = []
            documents_by_group[group].append(document)

        # process documents by group
        documents = []
        for group in documents_by_group:
            group_documents = documents_by_group[group]

            if group in document_processors:
                processors = document_processors[group]
                for p in processors:
                    print('Running document processor {}'.format(p))
                    group_documents, resources = p(self.options).process(group_documents, resources)

            completed_documents = []
            chained_documents = []
            for document in group_documents:
                if group is '' or document.group == group:
                    completed_documents.append(document)
                else:
                    chained_documents.append(document)

            if len(chained_documents):
                chained_documents, resources = self.process_documents(chained_documents, resources)
                documents.extend(chained_documents)

            documents.extend(completed_documents)

        return documents, resources

    def process_resources(self, documents, resources):
        # break resources into groups by extension
        resources_by_group = {}
        for resource in resources:
            group = resource.extension
            if group not in resources_by_group:
                resources_by_group[group] = []
            resources_by_group[group].append(resource)

        # process resources by extension
        resources = []
        for group in resources_by_group:
            group_resources = resources_by_group[group]

            if group not in resource_processors:
                group = ''

            if group in resource_processors:
                p = resource_processors[group]
                print('Running resource processor {}'.format(p))
                documents, group_resources = p(self.options).process(documents, group_resources)

            completed_resources = []
            chained_resources = []
            for resource in group_resources:
                if group is '' or resource.extension == group:
                    completed_resources.append(resource)
                else:
                    chained_resources.append(resource)

            if len(chained_resources):
                documents, chained_resources = self.process_resources(documents, chained_resources)
                resources.extend(chained_resources)

            resources.extend(completed_resources)

        return documents, resources

    def process(self, documents, resources):
        # preprocess everything
        for p in preprocessors:
            print('Running pre-processor {}'.format(p))
            documents, resources = p(self.options).process(documents, resources)

        documents, resources = self.process_documents(documents, resources)
        documents, resources = self.process_resources(documents, resources)

        # break documents into groups by extension
        documents_by_group = {}
        for document in documents:
            group = document.extension
            if group not in documents_by_group:
                documents_by_group[group] = []
            documents_by_group[group].append(document)

        # render markup for all documents
        documents = []
        for extension in documents_by_group:
            group_documents = documents_by_group[extension]

            if extension not in markup_processors:
                extension = ''

            if extension in markup_processors:
                processors = markup_processors[extension]
                for p in processors:
                    print('Running markup processor for {}: {}'.format(document.path, p))
                    group_documents, resources = p(self.options).process(group_documents, resources)

            documents.extend(group_documents)

        # set default document uris
        for document in documents:
            if document.uri is None:
                document.uri = document.path + document.extension

        # postprocess everything
        for p in postprocessors:
            print('Running post-processor {}'.format(p))
            documents, resources = p(self.options).process(documents, resources)

        # finalize document uris
        for document in documents:
            document.uri = urljoin(self.options['site']['root'], document.uri)
            if not document.get('link'):
                document['link'] = urljoin(self.options['site']['uri'], document.uri)

        return documents, resources

    def write(self, documents, resources):
        render = Render(self.options, documents)

        for document in documents:
            filepath = path.join(self.output_path, document.path)
            filepath += document.extension

            print('Rendering document {}'.format(filepath))
            with open(filepath, 'w') as f:
                f.write(render.render(document))

        for resource in resources:
            filepath = path.join(self.output_path, resource.path)
            filepath += resource.extension

            print('Writing resource {}'.format(filepath))
            with open(filepath, 'wb') as f:
                f.write(resource.content)

    def create_output_hierarchy(self, documents, resources):
        hierarchy = set()

        for document in documents:
            dirname = path.dirname(document.path)
            hierarchy.add(path.join(self.output_path, dirname))

        for resource in resources:
            dirname = path.dirname(resource.path)
            hierarchy.add(path.join(self.output_path, dirname))

        print('Creating output hierarchy: {}'.format(hierarchy))

        for dir in hierarchy:
            os.makedirs(dir, exist_ok=True)

    def run(self):
        documents, resources = self.load()
        documents, resources = self.process(documents, resources)

        self.create_output_hierarchy(documents, resources)
        self.write(documents, resources)
<commit_msg>Small fixes for building documents<commit_after>import os
from os import path
from urllib.parse import urljoin

from nib import Document, Resource, Render
from nib.processor import preprocessors, postprocessors,\
    document_processors, resource_processors, markup_processors
import nib.plugins

class Build(object):
    def __init__(self, options):
        self.document_path = options['document_path']
        self.resource_path = options['resource_path']
        self.output_path = options['output_path']
        self.options = options

        nib.plugins.load(options)

    def load(self):
        documents = []

        for root, dirs, files in os.walk(self.document_path):
            for filename in files:
                filepath = path.join(root, filename)

                try:
                    print('Reading {}'.format(filepath))
                    document = Document.from_file(filepath, self.options)
                    documents.append(document)

                except Exception as e:
                    print('Error while reading {}: {}'.format(filepath, e))

        resources = []

        for root, dirs, files in os.walk(self.resource_path):
            for filename in files:
                filepath = path.join(root, filename)

                try:
                    print('Reading {}'.format(filepath))
                    resource = Resource.from_file(filepath, self.options)
                    resources.append(resource)

                except Exception as e:
                    print('Error while reading {}: {}'.format(filepath, e))

        return documents, resources

    def process_documents(self, documents, resources):
        # break documents into groups by type
        documents_by_group = {}
        for document in documents:
            group = document.group
            if group not in documents_by_group:
                documents_by_group[group] = []
            documents_by_group[group].append(document)

        # process documents by group
        documents = []
        for group in documents_by_group:
            group_documents = documents_by_group[group]

            if group in document_processors:
                processors = document_processors[group]
                for p in processors:
                    print('Running document processor {}'.format(p))
                    group_documents, resources = p(self.options).process(group_documents, resources)

            completed_documents = []
            chained_documents = []
            for document in group_documents:
                if group is '' or document.group == group:
                    completed_documents.append(document)
                else:
                    chained_documents.append(document)

            if len(chained_documents):
                chained_documents, resources = self.process_documents(chained_documents, resources)
                documents.extend(chained_documents)

            documents.extend(completed_documents)

        return documents, resources

    def process_resources(self, documents, resources):
        # break resources into groups by extension
        resources_by_group = {}
        for resource in resources:
            group = resource.extension
            if group not in resources_by_group:
                resources_by_group[group] = []
            resources_by_group[group].append(resource)

        # process resources by extension
        resources = []
        for group in resources_by_group:
            group_resources = resources_by_group[group]

            if group not in resource_processors:
                group = ''

            if group in resource_processors:
                p = resource_processors[group]
                print('Running resource processor {}'.format(p))
                documents, group_resources = p(self.options).process(documents, group_resources)

            completed_resources = []
            chained_resources = []
            for resource in group_resources:
                if group is '' or resource.extension == group:
                    completed_resources.append(resource)
                else:
                    chained_resources.append(resource)

            if len(chained_resources):
                documents, chained_resources = self.process_resources(documents, chained_resources)
                resources.extend(chained_resources)

            resources.extend(completed_resources)

        return documents, resources

    def process(self, documents, resources):
        # preprocess everything
        for p in preprocessors:
            print('Running pre-processor {}'.format(p))
            documents, resources = p(self.options).process(documents, resources)

        documents, resources = self.process_documents(documents, resources)
        documents, resources = self.process_resources(documents, resources)

        # break documents into groups by extension
        documents_by_group = {}
        for document in documents:
            group = document.extension
            if group not in documents_by_group:
                documents_by_group[group] = []
            documents_by_group[group].append(document)

        # render markup for all documents
        documents = []
        for extension in documents_by_group:
            group_documents = documents_by_group[extension]

            if extension not in markup_processors:
                extension = ''

            if extension in markup_processors:
                processors = markup_processors[extension]
                for p in processors:
                    print('Running markup processor {}'.format(p))
                    group_documents, resources = p(self.options).process(group_documents, resources)

            documents.extend(group_documents)

        # set default document uris
        for document in documents:
            if document.uri is None:
                document.uri = document.path + document.extension

        # postprocess everything
        for p in postprocessors:
            print('Running post-processor {}'.format(p))
            documents, resources = p(self.options).process(documents, resources)

        # finalize document uris
        for document in documents:
            document.uri = urljoin(self.options['site']['root'], document.uri)
            if not document.get('link'):
                document['link'] = document.uri

        return documents, resources

    def write(self, documents, resources):
        render = Render(self.options, documents)

        for document in documents:
            filepath = path.join(self.output_path, document.path)
            filepath += document.extension

            print('Rendering document {}'.format(filepath))
            with open(filepath, 'w') as f:
                f.write(render.render(document))

        for resource in resources:
            filepath = path.join(self.output_path, resource.path)
            filepath += resource.extension

            print('Writing resource {}'.format(filepath))
            with open(filepath, 'wb') as f:
                f.write(resource.content)

    def create_output_hierarchy(self, documents, resources):
        hierarchy = set()

        for document in documents:
            dirname = path.dirname(document.path)
            hierarchy.add(path.join(self.output_path, dirname))

        for resource in resources:
            dirname = path.dirname(resource.path)
            hierarchy.add(path.join(self.output_path, dirname))

        print('Creating output hierarchy: {}'.format(hierarchy))

        for dir in hierarchy:
            os.makedirs(dir, exist_ok=True)

    def run(self):
        documents, resources = self.load()
        documents, resources = self.process(documents, resources)

        self.create_output_hierarchy(documents, resources)
        self.write(documents, resources)
<|endoftext|>


<commit_before># This script is executed in the main console namespace so
# that all the variables defined here become console variables.

import ddapp

import os
import sys
import vtk
import PythonQt
from PythonQt import QtCore, QtGui
import ddapp.applogic as app
from ddapp import matlab
from ddapp import jointcontrol
from ddapp import cameracontrol
from ddapp import ik
from ddapp import ikeditor
from ddapp import objectmodel as om
from ddapp import spreadsheet
from ddapp import tdx
from ddapp import perception
from ddapp import segmentation
from ddapp import cameraview
from ddapp import colorize
from ddapp import robotstate
from ddapp import footstepsdriver
from ddapp import footstepsdriverpanel
from ddapp import atlasdriver
from ddapp import atlasdriverpanel
from ddapp import robotplanlistener
from ddapp import vtkNumpy as vnp
from ddapp import visualization as vis
from ddapp import actionhandlers
from ddapp.timercallback import TimerCallback
from ddapp import segmentationpanel
from ddapp import lcmUtils
from ddapp.shallowCopy import shallowCopy
import drc as lcmdrc

import functools

import numpy as np
from ddapp.debugVis import DebugData
from ddapp import ioUtils as io

app.startup(globals())
om.init(app.getMainWindow().objectTree(), app.getMainWindow().propertiesPanel())
actionhandlers.init()

quit = app.quit
exit = quit
view = app.getDRCView()
camera = view.camera()
tree = app.getMainWindow().objectTree()
orbit = cameracontrol.OrbitController(view)
showPolyData = segmentation.showPolyData
updatePolyData = segmentation.updatePolyData

###############################################################################

useIk = True
usePerception = False
useSpreadsheet = True
useFootsteps = True
usePlanning = False
useAtlasDriver = False

poseCollection = PythonQt.dd.ddSignalMap()
costCollection = PythonQt.dd.ddSignalMap()

if useSpreadsheet:
    spreadsheet.init(poseCollection, costCollection)

if useIk:

    ikview = app.getViewManager().createView('IK View', 'VTK View')

    app.getViewManager().switchToView('IK View')

    ikFolder = om.addContainer('Drake IK')
    om.addPlaceholder('matlab server', om.Icons.Matlab, ikFolder)

    #urdfFile = os.path.join(app.getDRCBase(), 'software/models/mit_gazebo_models/mit_robot_drake/model_minimal_contact_fixedjoint_hands.urdf')
    urdfFile = os.path.join(app.getDRCBase(), 'software/models/mit_gazebo_models/mit_robot/model_LI_RR.urdf')

    model = app.loadRobotModelFromFile(urdfFile)
    obj = om.addRobotModel(model, ikFolder)
    obj.addToView(ikview)
    defaultRobotModel = obj

    useTable = False
    if useTable:
        tableModel = ikview.loadURDFModel(os.path.join(app.getDRCBase(), 'software/drake/systems/plants/test/table.urdf'))
        tableModel.setVisible(True)
        affordancesFolder = om.getOrCreateContainer('affordances')
        om.addRobotModel(tableModel, affordancesFolder)

    jc = jointcontrol.JointController([model], poseCollection)
    jc.addNominalPoseFromFile(app.getNominalPoseMatFile())
    jc.setNominalPose()
    jc.addPose('q_end', jc.poses['q_nom'])
    jc.addPose('q_start', jc.poses['q_nom'])

    def startIkServer():
        s = ik.AsyncIKCommunicator(jc)
        s.outputConsole = app.getOutputConsole()
        s.infoFunc = app.displaySnoptInfo
        s.start()
        s.startServerAsync()

        e = ikeditor.IKEditor(app.getMainWindow(), s, poseCollection, costCollection)
        e.makeFrameWidget(ikview)
        app.addWidgetToDock(e.widget)
        tdx.init(ikview, e)

    startAutomatically = False
    if startAutomatically:
        startIkServer()

    app.resetCamera(viewDirection=[-1,0,0], view=ikview)

if useAtlasDriver:
    atlasdriver.init()
    atlasdriverpanel.init(atlasdriver.driver)

if useFootsteps:
    # footsteps.init()
    footstepsdriver.init(jc)
    footstepsdriverpanel.init(footstepsdriver.driver)

if usePlanning:
    planListener = robotplanlistener.RobotPlanListener()
    planListener.playbackSpeed = 5.0

    def planCallback():
        planListener.stopAnimation()
        planListener.playPlan(jc)
        planListener.picklePlan('plan.pkl')

    def animationCallback():
        sendEstRobotState(jc.currentPoseName)

    planListener.manipPlanCallback = planCallback
    planListener.animationCallback = animationCallback

    app.addToolbarMacro('plot plan', planListener.plotPlan)

if usePerception:

    mitRobotDir = os.path.join(app.getDRCBase(), 'software/models/mit_gazebo_models/mit_robot')
    urdfFile = os.path.join(mitRobotDir, 'model_LI_RR.urdf')

    robotStateModel = app.loadRobotModelFromFile(urdfFile)

    robotStateJointController = jointcontrol.JointController([robotStateModel])
    robotStateJointController.setZeroPose()

    perception.init(view, robotStateJointController)
    segmentationpanel.init()
    cameraview.init()
    colorize.init()

    sensorsFolder = om.getOrCreateContainer('sensors')
    robotStateModel = om.addRobotModel(robotStateModel, sensorsFolder)
    robotStateModel.addToView(view)
    defaultRobotModel = robotStateModel

    cameraview.cameraView.rayCallback = segmentation.extractPointsAlongClickRay

    def grabRobotState():
        poseName = 'EST_ROBOT_STATE'
        robotStatePose = robotStateJointController.poses[poseName]
        s.sendPoseToServer(robotStatePose, poseName)
        s.forcePose(poseName)

    def onRobotModel(m):
        model = app.loadRobotModelFromString(m.urdf_xml_string)
        sensorsFolder = om.getOrCreateContainer('sensors')
        obj = om.addRobotModel(model, sensorsFolder)
        obj.setProperty('Name', 'model publisher')
        robotStateJointController.models.append(model)

        global robotStateModel
        obj.addToView(robotStateModel.views[0])
        robotStateModel.setProperty('Visible', False)
        robotStateModel = obj

    lcmUtils.captureMessageCallback('ROBOT_MODEL', lcmdrc.robot_urdf_t, onRobotModel)

    def setupFinger(pos):

        grabRobotState()
        p = perception._multisenseItem.model.revPolyData
        vis.showPolyData(p, 'lidar', alpha=0.3)
        t = s.grabCurrentLinkPose('l_hand')
        vis.showFrame(t, 'lhand')

        fingerTip = np.array(pos)

        handLinkPos = np.array(t.GetPosition())
        posOffset = fingerTip - handLinkPos

        ft = vtk.vtkTransform()
        ft.DeepCopy(t)

        ft.PostMultiply()
        ft.Translate(posOffset)

        vis.showFrame(ft, 'fingerTip')

        hand_to_finger = vtk.vtkTransform()
        hand_to_finger.DeepCopy(ft)
        hand_to_finger.Concatenate(t.GetLinearInverse())

        posOffset = np.array(hand_to_finger.GetPosition())

        s.setPointInLink('l_hand', posOffset)

app.resetCamera(viewDirection=[-1,0,0], view=view)

def affUpdaterOn():
    vis.affup.updater.on()

def affUpdaterOff():
    vis.affup.updater.off()

def getLinkFrame(linkName, model=None):
    model = model or defaultRobotModel
    return model.getLinkFrame(linkName)

def showLinkFrame(linkName, model=None):
    return vis.updateFrame(getLinkFrame(linkName, model), linkName, parent='link frames')

def createWalkingGoal():
    footstepsdriver.driver.createWalkingGoal(defaultRobotModel)

def resetCameraToRobot():
    t = getLinkFrame('utorso')
    focalPoint = [0.3, 0.0, 0.3]
    position = [-4.0, -2.0, 2.0]
    t.TransformPoint(focalPoint, focalPoint)
    t.TransformPoint(position, position)
    c = view.camera()
    c.SetFocalPoint(focalPoint)
    c.SetPosition(position)
    c.SetViewUp([0.0, 0.0, 1.0])
    view.render()

def resetCameraToHeadView():

    head = getLinkFrame('head')
    utorso = getLinkFrame('utorso')

    viewDirection = np.array([1.0, 0.0, 0.0])
    utorso.TransformVector(viewDirection, viewDirection)

    cameraPosition = np.array(head.GetPosition()) + 0.10 * viewDirection

    camera = view.camera()

    focalOffset = np.array(camera.GetFocalPoint()) - np.array(camera.GetPosition())
    focalOffset /= np.linalg.norm(focalOffset)

    camera.SetPosition(cameraPosition)
    camera.SetFocalPoint(cameraPosition + focalOffset*0.03)
    camera.SetViewUp([0, 0, 1])
    camera.SetViewAngle(90)
    view.render()

def sendEstRobotState(poseName='q_end'):
    pose = jc.getPose(poseName)
    msg = robotstate.drakePoseToRobotState(pose)
    lcmUtils.publish('EST_ROBOT_STATE', msg)

tc = TimerCallback()
tc.targetFps = 60
tc.callback = resetCameraToHeadView

class ViewEventFilter(object):

    def __init__(self, view):
        self.view = view
        self.initEventFilter()
        self.doubleClickCallback = None

    def filterEvent(self, obj, event):
        if event.type() == QtCore.QEvent.MouseButtonDblClick:
            if self.doubleClickCallback:
                result = self.doubleClickCallback(vis.mapMousePosition(obj, event))
                self.eventFilter.setEventHandlerResult(result)

    def initEventFilter(self):
        self.eventFilter = PythonQt.dd.ddPythonEventFilter()
        qvtkwidget = self.view.vtkWidget()
        qvtkwidget.installEventFilter(self.eventFilter)
        self.eventFilter.addFilteredEventType(QtCore.QEvent.MouseButtonDblClick)
        self.eventFilter.connect('handleEvent(QObject*, QEvent*)', self.filterEvent)

def onViewDoubleClicked(displayPoint):

    model = defaultRobotModel.model

    polyData, pickedPoint = vis.pickPoint(displayPoint, view=ikview, pickType='cells')
    linkName = model.getLinkNameForMesh(polyData)
    if not linkName:
        return False

    colorNoHighlight = QtGui.QColor(190, 190, 190)
    colorHighlight = QtCore.Qt.red

    linkNames = [linkName]
    model.setColor(colorNoHighlight)

    for name in linkNames:
        model.setLinkColor(name, colorHighlight)

    return True

ef = ViewEventFilter(ikview)
ef.doubleClickCallback = onViewDoubleClicked
<commit_msg>fix showLinkFrame()<commit_after># This script is executed in the main console namespace so
# that all the variables defined here become console variables.

import ddapp

import os
import sys
import vtk
import PythonQt
from PythonQt import QtCore, QtGui
import ddapp.applogic as app
from ddapp import matlab
from ddapp import jointcontrol
from ddapp import cameracontrol
from ddapp import ik
from ddapp import ikeditor
from ddapp import objectmodel as om
from ddapp import spreadsheet
from ddapp import tdx
from ddapp import perception
from ddapp import segmentation
from ddapp import cameraview
from ddapp import colorize
from ddapp import robotstate
from ddapp import footstepsdriver
from ddapp import footstepsdriverpanel
from ddapp import atlasdriver
from ddapp import atlasdriverpanel
from ddapp import robotplanlistener
from ddapp import vtkNumpy as vnp
from ddapp import visualization as vis
from ddapp import actionhandlers
from ddapp.timercallback import TimerCallback
from ddapp import segmentationpanel
from ddapp import lcmUtils
from ddapp.shallowCopy import shallowCopy
import drc as lcmdrc

import functools

import numpy as np
from ddapp.debugVis import DebugData
from ddapp import ioUtils as io

app.startup(globals())
om.init(app.getMainWindow().objectTree(), app.getMainWindow().propertiesPanel())
actionhandlers.init()

quit = app.quit
exit = quit
view = app.getDRCView()
camera = view.camera()
tree = app.getMainWindow().objectTree()
orbit = cameracontrol.OrbitController(view)
showPolyData = segmentation.showPolyData
updatePolyData = segmentation.updatePolyData

###############################################################################

useIk = True
usePerception = False
useSpreadsheet = True
useFootsteps = True
usePlanning = False
useAtlasDriver = False

poseCollection = PythonQt.dd.ddSignalMap()
costCollection = PythonQt.dd.ddSignalMap()

if useSpreadsheet:
    spreadsheet.init(poseCollection, costCollection)

if useIk:

    ikview = app.getViewManager().createView('IK View', 'VTK View')

    app.getViewManager().switchToView('IK View')

    ikFolder = om.addContainer('Drake IK')
    om.addPlaceholder('matlab server', om.Icons.Matlab, ikFolder)

    #urdfFile = os.path.join(app.getDRCBase(), 'software/models/mit_gazebo_models/mit_robot_drake/model_minimal_contact_fixedjoint_hands.urdf')
    urdfFile = os.path.join(app.getDRCBase(), 'software/models/mit_gazebo_models/mit_robot/model_LI_RR.urdf')

    model = app.loadRobotModelFromFile(urdfFile)
    obj = om.addRobotModel(model, ikFolder)
    obj.addToView(ikview)
    defaultRobotModel = obj

    useTable = False
    if useTable:
        tableModel = ikview.loadURDFModel(os.path.join(app.getDRCBase(), 'software/drake/systems/plants/test/table.urdf'))
        tableModel.setVisible(True)
        affordancesFolder = om.getOrCreateContainer('affordances')
        om.addRobotModel(tableModel, affordancesFolder)

    jc = jointcontrol.JointController([model], poseCollection)
    jc.addNominalPoseFromFile(app.getNominalPoseMatFile())
    jc.setNominalPose()
    jc.addPose('q_end', jc.poses['q_nom'])
    jc.addPose('q_start', jc.poses['q_nom'])

    def startIkServer():
        s = ik.AsyncIKCommunicator(jc)
        s.outputConsole = app.getOutputConsole()
        s.infoFunc = app.displaySnoptInfo
        s.start()
        s.startServerAsync()

        e = ikeditor.IKEditor(app.getMainWindow(), s, poseCollection, costCollection)
        e.makeFrameWidget(ikview)
        app.addWidgetToDock(e.widget)
        tdx.init(ikview, e)

    startAutomatically = False
    if startAutomatically:
        startIkServer()

    app.resetCamera(viewDirection=[-1,0,0], view=ikview)

if useAtlasDriver:
    atlasdriver.init()
    atlasdriverpanel.init(atlasdriver.driver)

if useFootsteps:
    # footsteps.init()
    footstepsdriver.init(jc)
    footstepsdriverpanel.init(footstepsdriver.driver)

if usePlanning:
    planListener = robotplanlistener.RobotPlanListener()
    planListener.playbackSpeed = 5.0

    def planCallback():
        planListener.stopAnimation()
        planListener.playPlan(jc)
        planListener.picklePlan('plan.pkl')

    def animationCallback():
        sendEstRobotState(jc.currentPoseName)

    planListener.manipPlanCallback = planCallback
    planListener.animationCallback = animationCallback

    app.addToolbarMacro('plot plan', planListener.plotPlan)

if usePerception:

    mitRobotDir = os.path.join(app.getDRCBase(), 'software/models/mit_gazebo_models/mit_robot')
    urdfFile = os.path.join(mitRobotDir, 'model_LI_RR.urdf')

    robotStateModel = app.loadRobotModelFromFile(urdfFile)

    robotStateJointController = jointcontrol.JointController([robotStateModel])
    robotStateJointController.setZeroPose()

    perception.init(view, robotStateJointController)
    segmentationpanel.init()
    cameraview.init()
    colorize.init()

    sensorsFolder = om.getOrCreateContainer('sensors')
    robotStateModel = om.addRobotModel(robotStateModel, sensorsFolder)
    robotStateModel.addToView(view)
    defaultRobotModel = robotStateModel

    cameraview.cameraView.rayCallback = segmentation.extractPointsAlongClickRay

    def grabRobotState():
        poseName = 'EST_ROBOT_STATE'
        robotStatePose = robotStateJointController.poses[poseName]
        s.sendPoseToServer(robotStatePose, poseName)
        s.forcePose(poseName)

    def onRobotModel(m):
        model = app.loadRobotModelFromString(m.urdf_xml_string)
        sensorsFolder = om.getOrCreateContainer('sensors')
        obj = om.addRobotModel(model, sensorsFolder)
        obj.setProperty('Name', 'model publisher')
        robotStateJointController.models.append(model)

        global robotStateModel
        obj.addToView(robotStateModel.views[0])
        robotStateModel.setProperty('Visible', False)
        robotStateModel = obj

    lcmUtils.captureMessageCallback('ROBOT_MODEL', lcmdrc.robot_urdf_t, onRobotModel)

    def setupFinger(pos):

        grabRobotState()
        p = perception._multisenseItem.model.revPolyData
        vis.showPolyData(p, 'lidar', alpha=0.3)
        t = s.grabCurrentLinkPose('l_hand')
        vis.showFrame(t, 'lhand')

        fingerTip = np.array(pos)

        handLinkPos = np.array(t.GetPosition())
        posOffset = fingerTip - handLinkPos

        ft = vtk.vtkTransform()
        ft.DeepCopy(t)

        ft.PostMultiply()
        ft.Translate(posOffset)

        vis.showFrame(ft, 'fingerTip')

        hand_to_finger = vtk.vtkTransform()
        hand_to_finger.DeepCopy(ft)
        hand_to_finger.Concatenate(t.GetLinearInverse())

        posOffset = np.array(hand_to_finger.GetPosition())

        s.setPointInLink('l_hand', posOffset)

app.resetCamera(viewDirection=[-1,0,0], view=view)

def affUpdaterOn():
    vis.affup.updater.on()

def affUpdaterOff():
    vis.affup.updater.off()

def getLinkFrame(linkName, model=None):
    model = model or defaultRobotModel
    return model.getLinkFrame(linkName)

def showLinkFrame(linkName, model=None):
    frame = getLinkFrame(linkName, model)
    if not frame:
        raise Exception('Link not found: ' + linkName)
    return vis.updateFrame(frame, linkName, parent='link frames')

def createWalkingGoal():
    footstepsdriver.driver.createWalkingGoal(defaultRobotModel)

def resetCameraToRobot():
    t = getLinkFrame('utorso')
    focalPoint = [0.3, 0.0, 0.3]
    position = [-4.0, -2.0, 2.0]
    t.TransformPoint(focalPoint, focalPoint)
    t.TransformPoint(position, position)
    c = view.camera()
    c.SetFocalPoint(focalPoint)
    c.SetPosition(position)
    c.SetViewUp([0.0, 0.0, 1.0])
    view.render()

def resetCameraToHeadView():

    head = getLinkFrame('head')
    utorso = getLinkFrame('utorso')

    viewDirection = np.array([1.0, 0.0, 0.0])
    utorso.TransformVector(viewDirection, viewDirection)

    cameraPosition = np.array(head.GetPosition()) + 0.10 * viewDirection

    camera = view.camera()

    focalOffset = np.array(camera.GetFocalPoint()) - np.array(camera.GetPosition())
    focalOffset /= np.linalg.norm(focalOffset)

    camera.SetPosition(cameraPosition)
    camera.SetFocalPoint(cameraPosition + focalOffset*0.03)
    camera.SetViewUp([0, 0, 1])
    camera.SetViewAngle(90)
    view.render()

def sendEstRobotState(poseName='q_end'):
    pose = jc.getPose(poseName)
    msg = robotstate.drakePoseToRobotState(pose)
    lcmUtils.publish('EST_ROBOT_STATE', msg)

tc = TimerCallback()
tc.targetFps = 60
tc.callback = resetCameraToHeadView

class ViewEventFilter(object):

    def __init__(self, view):
        self.view = view
        self.initEventFilter()
        self.doubleClickCallback = None

    def filterEvent(self, obj, event):
        if event.type() == QtCore.QEvent.MouseButtonDblClick:
            if self.doubleClickCallback:
                result = self.doubleClickCallback(vis.mapMousePosition(obj, event))
                self.eventFilter.setEventHandlerResult(result)

    def initEventFilter(self):
        self.eventFilter = PythonQt.dd.ddPythonEventFilter()
        qvtkwidget = self.view.vtkWidget()
        qvtkwidget.installEventFilter(self.eventFilter)
        self.eventFilter.addFilteredEventType(QtCore.QEvent.MouseButtonDblClick)
        self.eventFilter.connect('handleEvent(QObject*, QEvent*)', self.filterEvent)

def onViewDoubleClicked(displayPoint):

    model = defaultRobotModel.model

    polyData, pickedPoint = vis.pickPoint(displayPoint, view=ikview, pickType='cells')
    linkName = model.getLinkNameForMesh(polyData)
    if not linkName:
        return False

    colorNoHighlight = QtGui.QColor(190, 190, 190)
    colorHighlight = QtCore.Qt.red

    linkNames = [linkName]
    model.setColor(colorNoHighlight)

    for name in linkNames:
        model.setLinkColor(name, colorHighlight)

    return True

ef = ViewEventFilter(ikview)
ef.doubleClickCallback = onViewDoubleClicked
<|endoftext|>


<commit_before>import util, requests, csv, os

API_ENDPOINT = "http://data.beeldengeluid.nl/api/"
SCHEME_ENDPOINT = "http://data.beeldengeluid.nl/gtaa/%s"
SCHEMES = (
    "GeografischeNamen",
    "Geografisch",
    "Namen",
    "Persoonsnamen",
    "OnderwerpenBenG",
    "Onderwerpen",
    "Maker",
    "Genre"
)

def _format_concept(concept):
    hiddenLabel = concept.get("hiddenLabel", [None])
    prefLabel = concept.get("prefLabel", [None])
    scheme = concept.get("inScheme", [None])
    id_ = concept.get("notation", [None])

    return {
        "label" : hiddenLabel[0] if hiddenLabel[0] else prefLabel[0],
        "hiddenLabel" : hiddenLabel[0],
        "prefLabel" : prefLabel[0],
        "uuid" : concept.get("uuid", None),
        "scheme" : scheme[0],
        "id" : id_[0],
        "deleted" : concept.get("deleted", None),
        "uri" : concept.get("uri", None),
        "tenant" : concept.get("tenant", None)
    }

def lookupcombined(q, qtype):
    if qtype not in ("gtaa", "wikidata"):
        raise Exception("Invalid query type")

    csvfile = os.path.dirname(__file__) + "/combined.csv"

    for row in csv.DictReader(open(csvfile)):
        if row[qtype] == q:
            return row

    return False

def lookup(id_):
    url = SCHEME_ENDPOINT % id_ + ".json"
    req = requests.get(url)

    if req.status_code == 404:
        return False

    return _format_concept(req.json())

def findconcepts(q, inScheme = None):
    endpoint = API_ENDPOINT + "find-concepts"

    if inScheme:
        if not inScheme in SCHEMES:
            raise Exception("Unknown scheme")

        scheme = SCHEME_ENDPOINT % inScheme
        q += ' AND inScheme:"%s"' % scheme

    r = util.apirequest(endpoint, {
        "q" : q,
        "format" : "json"
    })

    if not "response" in r:
        return False

    if not "docs" in r["response"] or len(r["response"]["docs"]) == 0:
        return False

    return map(_format_concept, r["response"]["docs"])<commit_msg>Adding an awful hack for the 'Makers' scheme<commit_after>import util, requests, csv, os

API_ENDPOINT = "http://data.beeldengeluid.nl/api/"
SCHEME_ENDPOINT = "http://data.beeldengeluid.nl/gtaa/%s"
SCHEMES = (
    "GeografischeNamen",
    "Geografisch",
    "Namen",
    "Persoonsnamen",
    "OnderwerpenBenG",
    "Onderwerpen",
    "Maker",
    "Makers", # This is a hack, see the code below
    "Genre"
)

def _format_concept(concept):
    hiddenLabel = concept.get("hiddenLabel", [None])
    prefLabel = concept.get("prefLabel", [None])
    scheme = concept.get("inScheme", [None])
    id_ = concept.get("notation", [None])

    return {
        "label" : hiddenLabel if hiddenLabel else prefLabel,
        "hiddenLabel" : hiddenLabel,
        "prefLabel" : prefLabel,
        "uuid" : concept.get("uuid", None),
        "scheme" : scheme[0],
        "id" : id_[0],
        "deleted" : concept.get("deleted", None),
        "uri" : concept.get("uri", None),
        "tenant" : concept.get("tenant", None),
        "scopeNote" : concept.get("scopeNote", None)
    }

def lookupcombined(q, qtype):
    if qtype not in ("gtaa", "wikidata"):
        raise Exception("Invalid query type")

    csvfile = os.path.dirname(__file__) + "/combined.csv"

    for row in csv.DictReader(open(csvfile)):
        if row[qtype] == q:
            return row

    return False

def lookup(id_):
    url = SCHEME_ENDPOINT % id_ + ".json"
    req = requests.get(url)

    if req.status_code == 404:
        return False

    return _format_concept(req.json())

def findconcepts(q, inScheme = None):
    endpoint = API_ENDPOINT + "find-concepts"

    if inScheme:
        if not inScheme in SCHEMES:
            raise Exception("Unknown scheme")

        # Awful kludge, see below
        if inScheme == "Makers":
            realScheme = "Persoonsnamen"
        else:
            realScheme = inScheme

        scheme = SCHEME_ENDPOINT % realScheme
        q += ' AND inScheme:"%s"' % scheme

    r = util.apirequest(endpoint, {
        "q" : q,
        "format" : "json"
    })

    if not "response" in r:
        return False

    if not "docs" in r["response"] or len(r["response"]["docs"]) == 0:
        return False

    # "Makers" has been merged with "Persoonsnamen", so we need to use this
    # ugly kludge to get the actual maker
    if inScheme == "Makers":
        docs = []

        for d in r["response"]["docs"]:
            if "changeNote" not in d:
                continue

            if "Samengevoegd met: Maker" not in d["changeNote"][0]:
                continue

            docs.append(d)
    else:
        docs = r["response"]["docs"]

    return map(_format_concept, docs)<|endoftext|>


<commit_before>"""
sentry.tasks.fetch_source
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

import itertools
import logging
import hashlib
import sourcemap
import urllib2
import zlib
from collections import namedtuple
from urlparse import urljoin

from django.utils.simplejson import JSONDecodeError
from sentry.utils.cache import cache

BAD_SOURCE = -1

# number of surrounding lines (on each side) to fetch
LINES_OF_CONTEXT = 5

UrlResult = namedtuple('UrlResult', ['url', 'headers', 'body'])

logger = logging.getLogger(__name__)

def trim_line(line):
    line = line.strip('\n')
    if len(line) > 150:
        line = line[:140] + ' [... truncated]'
    return line

def get_source_context(source, lineno, context=LINES_OF_CONTEXT):
    # lineno's in JS are 1-indexed
    # just in case. sometimes math is hard
    if lineno > 0:
        lineno -= 1

    lower_bound = max(0, lineno - context)
    upper_bound = min(lineno + 1 + context, len(source))

    try:
        pre_context = map(trim_line, source[lower_bound:lineno])
    except IndexError:
        pre_context = []

    try:
        context_line = trim_line(source[lineno])
    except IndexError:
        context_line = ''

    try:
        post_context = map(trim_line, source[(lineno + 1):upper_bound])
    except IndexError:
        post_context = []

    return pre_context, context_line, post_context

def discover_sourcemap(result):
    """
    Given a UrlResult object, attempt to discover a sourcemap.
    """
    # When coercing the headers returned by urllib to a dict
    # all keys become lowercase so they're normalized
    map_path = result.headers.get('sourcemap', result.headers.get('x-sourcemap'))
    if not map_path:
        map_path = sourcemap.discover(result.body)

    if map_path:
        # ensure url is absolute
        map_path = urljoin(result.url, map_path)

    return map_path

def fetch_url_content(url):
    """
    Pull down a URL, returning a tuple (url, headers, body).
    """
    import sentry

    try:
        opener = urllib2.build_opener()
        opener.addheaders = [('User-Agent', 'Sentry/%s' % sentry.VERSION)]
        req = opener.open(url)
        headers = dict(req.headers)
        body = req.read()
        if headers.get('content-encoding') == 'gzip':
            # Content doesn't *have* to respect the Accept-Encoding header
            # and may send gzipped data regardless.
            # See: http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk/2424549#2424549
            body = zlib.decompress(body, 16 + zlib.MAX_WBITS)
        body = body.rstrip('\n')
    except Exception:
        return BAD_SOURCE

    return (url, headers, body)

def fetch_url(url):
    """
    Pull down a URL, returning a UrlResult object.

    Attempts to fetch from the cache.
    """

    cache_key = 'fetch_url:v2:%s' % (
        hashlib.md5(url.encode('utf-8')).hexdigest(),)
    result = cache.get(cache_key)
    if result is None:
        result = fetch_url_content(url)

        cache.set(cache_key, result, 60 * 5)

    if result == BAD_SOURCE:
        return result

    return UrlResult(*result)

def fetch_sourcemap(url):
    result = fetch_url(url)
    if result == BAD_SOURCE:
        return

    body = result.body
    # According to spec (https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit#heading=h.h7yy76c5il9v)
    # A SourceMap may be prepended with ")]}'" to cause a Javascript error.
    # If the file starts with that string, ignore the entire first line.
    if body.startswith(")]}'"):
        body = body.split('\n', 1)[1]
    try:
        index = sourcemap.loads(body)
    except (JSONDecodeError, ValueError):
        pass
    else:
        return index

def expand_javascript_source(data, **kwargs):
    """
    Attempt to fetch source code for javascript frames.

    Frames must match the following requirements:

    - lineno >= 0
    - colno >= 0
    - abs_path is the HTTP URI to the source
    - context_line is empty

    Mutates the input ``data`` with expanded context if available.
    """
    from sentry.interfaces import Stacktrace

    # TODO: clean this up
    legacy_style = ('sentry.interfaces.Stacktrace' in data)

    if legacy_style:
        stacktraces = [Stacktrace(**data['sentry.interfaces.Stacktrace'])]
    else:
        try:
            stacktraces = [
                Stacktrace(**e['stacktrace'])
                for e in data['sentry.interfaces.Exception']['values']
            ]
        except KeyError:
            stacktraces = []

    if not stacktraces:
        logger.debug('No stacktrace for event %r', data['event_id'])
        return

    # build list of frames that we can actually grab source for
    frames = []
    for stacktrace in stacktraces:
        frames.extend([
            f for f in stacktrace.frames
            if f.lineno is not None
            and f.is_url()
        ])

    if not frames:
        logger.debug('Event %r has no frames with enough context to fetch remote source', data['event_id'])
        return data

    file_list = set()
    sourcemap_capable = set()
    source_code = {}
    sourcemap_idxs = {}

    for f in frames:
        file_list.add(f.abs_path)
        if f.colno is not None:
            sourcemap_capable.add(f.abs_path)

    while file_list:
        filename = file_list.pop()

        # TODO: respect cache-contro/max-age headers to some extent
        logger.debug('Fetching remote source %r', filename)
        result = fetch_url(filename)

        if result == BAD_SOURCE:
            logger.debug('Unable to fetch remote source for %r', filename)
            continue

        # If we didn't have a colno, a sourcemap wont do us any good
        if filename not in sourcemap_capable:
            source_code[filename] = (result.body.splitlines(), None)
            continue

        map_path = discover_sourcemap(result)
        source_code[filename] = (result.body.splitlines(), map_path)
        if map_path:
            logger.debug('Found sourcemap %r for minified script %r', map_path, result.url)
        # If we've already fetched this sourcemap, move along
        elif map_path in sourcemap_idxs:
            continue
        else:
            logger.debug('No sourcemap found for %r', filename)
            continue

        index = fetch_sourcemap(map_path)
        if not index:
            sourcemap_idxs[sourcemap] = None
            continue

        sourcemap_idxs[sourcemap] = index

        # queue up additional source files for download
        for source in index.sources:
            if source not in source_code:
                file_list.add(urljoin(result.url, source))

    has_changes = False
    for frame in frames:
        try:
            source, map_path = source_code[frame.abs_path]
        except KeyError:
            # we must've failed pulling down the source
            continue

        # may have had a failure pulling down the sourcemap previously
        if sourcemap_idxs.get(map_path) and frame.colno is not None:
            token = sourcemap_idxs[map_path].lookup(frame.lineno, frame.colno)
            # TODO: is this urljoin right? (is it relative to the sourcemap or the originating file)
            abs_path = urljoin(map_path, token.src)
            logger.debug('Mapping compressed source %r to mapping in %r', frame.abs_path, abs_path)
            try:
                source, _ = source_code[abs_path]
            except KeyError:
                pass
            else:
                # Store original data in annotation
                frame.data = {
                    'orig_lineno': frame['lineno'],
                    'orig_colno': frame['colno'],
                    'orig_function': frame['function'],
                    'orig_abs_path': frame['abs_path'],
                    'orig_filename': frame['filename'],
                    'sourcemap': map_path,
                }

                # SourceMap's return zero-indexed lineno's
                frame.lineno = token.src_line + 1
                frame.colno = token.src_col
                frame.function = token.name
                frame.abs_path = abs_path
                frame.filename = token.src

        has_changes = True

        # TODO: theoretically a minified source could point to another mapped, minified source
        frame.pre_context, frame.context_line, frame.post_context = get_source_context(
            source=source, lineno=frame.lineno)

    if has_changes:
        if legacy_style:
            data['sentry.interfaces.Stacktrace'] = stacktraces[0].serialize()
        else:
            for exception, stacktrace in itertools.izip(data['sentry.interfaces.Exception']['values'], stacktraces):
                exception['stacktrace'] = stacktrace.serialize()
<commit_msg>Revert "Remove excess logger statement and correct load call"<commit_after>"""
sentry.tasks.fetch_source
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

import itertools
import logging
import hashlib
import sourcemap
import urllib2
import zlib
from collections import namedtuple
from urlparse import urljoin

from django.utils.simplejson import JSONDecodeError
from sentry.utils.cache import cache

BAD_SOURCE = -1

# number of surrounding lines (on each side) to fetch
LINES_OF_CONTEXT = 5

UrlResult = namedtuple('UrlResult', ['url', 'headers', 'body'])

logger = logging.getLogger(__name__)

def trim_line(line):
    line = line.strip('\n')
    if len(line) > 150:
        line = line[:140] + ' [... truncated]'
    return line

def get_source_context(source, lineno, context=LINES_OF_CONTEXT):
    # lineno's in JS are 1-indexed
    # just in case. sometimes math is hard
    if lineno > 0:
        lineno -= 1

    lower_bound = max(0, lineno - context)
    upper_bound = min(lineno + 1 + context, len(source))

    try:
        pre_context = map(trim_line, source[lower_bound:lineno])
    except IndexError:
        pre_context = []

    try:
        context_line = trim_line(source[lineno])
    except IndexError:
        context_line = ''

    try:
        post_context = map(trim_line, source[(lineno + 1):upper_bound])
    except IndexError:
        post_context = []

    return pre_context, context_line, post_context

def discover_sourcemap(result):
    """
    Given a UrlResult object, attempt to discover a sourcemap.
    """
    # When coercing the headers returned by urllib to a dict
    # all keys become lowercase so they're normalized
    map_path = result.headers.get('sourcemap', result.headers.get('x-sourcemap'))
    if not map_path:
        map_path = sourcemap.discover(result.body)

    if map_path:
        # ensure url is absolute
        map_path = urljoin(result.url, map_path)

    return map_path

def fetch_url_content(url):
    """
    Pull down a URL, returning a tuple (url, headers, body).
    """
    import sentry

    try:
        opener = urllib2.build_opener()
        opener.addheaders = [('User-Agent', 'Sentry/%s' % sentry.VERSION)]
        req = opener.open(url)
        headers = dict(req.headers)
        body = req.read()
        if headers.get('content-encoding') == 'gzip':
            # Content doesn't *have* to respect the Accept-Encoding header
            # and may send gzipped data regardless.
            # See: http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk/2424549#2424549
            body = zlib.decompress(body, 16 + zlib.MAX_WBITS)
        body = body.rstrip('\n')
    except Exception:
        return BAD_SOURCE

    return (url, headers, body)

def fetch_url(url):
    """
    Pull down a URL, returning a UrlResult object.

    Attempts to fetch from the cache.
    """

    cache_key = 'fetch_url:v2:%s' % (
        hashlib.md5(url.encode('utf-8')).hexdigest(),)
    result = cache.get(cache_key)
    if result is None:
        result = fetch_url_content(url)

        cache.set(cache_key, result, 60 * 5)

    if result == BAD_SOURCE:
        return result

    return UrlResult(*result)

def fetch_sourcemap(url):
    result = fetch_url(url)
    if result == BAD_SOURCE:
        return

    body = result.body
    # According to spec (https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit#heading=h.h7yy76c5il9v)
    # A SourceMap may be prepended with ")]}'" to cause a Javascript error.
    # If the file starts with that string, ignore the entire first line.
    if body.startswith(")]}'"):
        body = body.split('\n', 1)[1]
    try:
        index = sourcemap.load(body)
    except (JSONDecodeError, ValueError):
        if logger:
            logger.warning('Failed parsing sourcemap JSON: %r', body[:15], exc_info=True)
    else:
        return index

def expand_javascript_source(data, **kwargs):
    """
    Attempt to fetch source code for javascript frames.

    Frames must match the following requirements:

    - lineno >= 0
    - colno >= 0
    - abs_path is the HTTP URI to the source
    - context_line is empty

    Mutates the input ``data`` with expanded context if available.
    """
    from sentry.interfaces import Stacktrace

    # TODO: clean this up
    legacy_style = ('sentry.interfaces.Stacktrace' in data)

    if legacy_style:
        stacktraces = [Stacktrace(**data['sentry.interfaces.Stacktrace'])]
    else:
        try:
            stacktraces = [
                Stacktrace(**e['stacktrace'])
                for e in data['sentry.interfaces.Exception']['values']
            ]
        except KeyError:
            stacktraces = []

    if not stacktraces:
        logger.debug('No stacktrace for event %r', data['event_id'])
        return

    # build list of frames that we can actually grab source for
    frames = []
    for stacktrace in stacktraces:
        frames.extend([
            f for f in stacktrace.frames
            if f.lineno is not None
            and f.is_url()
        ])

    if not frames:
        logger.debug('Event %r has no frames with enough context to fetch remote source', data['event_id'])
        return data

    file_list = set()
    sourcemap_capable = set()
    source_code = {}
    sourcemap_idxs = {}

    for f in frames:
        file_list.add(f.abs_path)
        if f.colno is not None:
            sourcemap_capable.add(f.abs_path)

    while file_list:
        filename = file_list.pop()

        # TODO: respect cache-contro/max-age headers to some extent
        logger.debug('Fetching remote source %r', filename)
        result = fetch_url(filename)

        if result == BAD_SOURCE:
            logger.debug('Unable to fetch remote source for %r', filename)
            continue

        # If we didn't have a colno, a sourcemap wont do us any good
        if filename not in sourcemap_capable:
            source_code[filename] = (result.body.splitlines(), None)
            continue

        map_path = discover_sourcemap(result)
        source_code[filename] = (result.body.splitlines(), map_path)
        if map_path:
            logger.debug('Found sourcemap %r for minified script %r', map_path, result.url)
        # If we've already fetched this sourcemap, move along
        elif map_path in sourcemap_idxs:
            continue
        else:
            logger.debug('No sourcemap found for %r', filename)
            continue

        index = fetch_sourcemap(map_path)
        if not index:
            sourcemap_idxs[sourcemap] = None
            continue

        sourcemap_idxs[sourcemap] = index

        # queue up additional source files for download
        for source in index.sources:
            if source not in source_code:
                file_list.add(urljoin(result.url, source))

    has_changes = False
    for frame in frames:
        try:
            source, map_path = source_code[frame.abs_path]
        except KeyError:
            # we must've failed pulling down the source
            continue

        # may have had a failure pulling down the sourcemap previously
        if sourcemap_idxs.get(map_path) and frame.colno is not None:
            token = sourcemap_idxs[map_path].lookup(frame.lineno, frame.colno)
            # TODO: is this urljoin right? (is it relative to the sourcemap or the originating file)
            abs_path = urljoin(map_path, token.src)
            logger.debug('Mapping compressed source %r to mapping in %r', frame.abs_path, abs_path)
            try:
                source, _ = source_code[abs_path]
            except KeyError:
                pass
            else:
                # Store original data in annotation
                frame.data = {
                    'orig_lineno': frame['lineno'],
                    'orig_colno': frame['colno'],
                    'orig_function': frame['function'],
                    'orig_abs_path': frame['abs_path'],
                    'orig_filename': frame['filename'],
                    'sourcemap': map_path,
                }

                # SourceMap's return zero-indexed lineno's
                frame.lineno = token.src_line + 1
                frame.colno = token.src_col
                frame.function = token.name
                frame.abs_path = abs_path
                frame.filename = token.src

        has_changes = True

        # TODO: theoretically a minified source could point to another mapped, minified source
        frame.pre_context, frame.context_line, frame.post_context = get_source_context(
            source=source, lineno=frame.lineno)

    if has_changes:
        if legacy_style:
            data['sentry.interfaces.Stacktrace'] = stacktraces[0].serialize()
        else:
            for exception, stacktrace in itertools.izip(data['sentry.interfaces.Exception']['values'], stacktraces):
                exception['stacktrace'] = stacktrace.serialize()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Natural Language Toolkit: ChrF score
#
# Copyright (C) 2001-2018 NLTK Project
# Authors: Maja Popovic
# Contributors: Liling Tan, Aleš Tamchyna (Memsource)
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

""" ChrF score implementation """
from __future__ import division
from collections import Counter, defaultdict
import re

from nltk.util import ngrams

def sentence_chrf(reference, hypothesis, min_len=1, max_len=6, beta=3.0,
                  ignore_whitespace=True):
    """
    Calculates the sentence level CHRF (Character n-gram F-score) described in
     - Maja Popovic. 2015. CHRF: Character n-gram F-score for Automatic MT Evaluation.
       In Proceedings of the 10th Workshop on Machine Translation.
       http://www.statmt.org/wmt15/pdf/WMT49.pdf
     - Maja Popovic. 2016. CHRF Deconstructed: β Parameters and n-gram Weights.
       In Proceedings of the 1st Conference on Machine Translation.
       http://www.statmt.org/wmt16/pdf/W16-2341.pdf

    This implementation of CHRF only supports a single reference at the moment.

    For details not reported in the paper, consult Maja Popovic's original
    implementation: https://github.com/m-popovic/chrF

    The code should output results equivalent to running CHRF++ with the
    following options: -nw 0 -b 3

    An example from the original BLEU paper
    http://www.aclweb.org/anthology/P02-1040.pdf

        >>> ref1 = str('It is a guide to action that ensures that the military '
        ...            'will forever heed Party commands').split()
        >>> hyp1 = str('It is a guide to action which ensures that the military '
        ...            'always obeys the commands of the party').split()
        >>> hyp2 = str('It is to insure the troops forever hearing the activity '
        ...            'guidebook that party direct').split()
        >>> sentence_chrf(ref1, hyp1) # doctest: +ELLIPSIS
        0.6349...
        >>> sentence_chrf(ref1, hyp2) # doctest: +ELLIPSIS
        0.3330...

    The infamous "the the the ... " example

        >>> ref = 'the cat is on the mat'.split()
        >>> hyp = 'the the the the the the the'.split()
        >>> sentence_chrf(ref, hyp)  # doctest: +ELLIPSIS
        0.1468...

    An example to show that this function allows users to use strings instead of
    tokens, i.e. list(str) as inputs.

        >>> ref1 = str('It is a guide to action that ensures that the military '
        ...            'will forever heed Party commands')
        >>> hyp1 = str('It is a guide to action which ensures that the military '
        ...            'always obeys the commands of the party')
        >>> sentence_chrf(ref1, hyp1) # doctest: +ELLIPSIS
        0.6349...
        >>> type(ref1) == type(hyp1) == str
        True
        >>> sentence_chrf(ref1.split(), hyp1.split()) # doctest: +ELLIPSIS
        0.6349...

    To skip the unigrams and only use 2- to 3-grams:

        >>> sentence_chrf(ref1, hyp1, min_len=2, max_len=3) # doctest: +ELLIPSIS
        0.6617...

    :param references: reference sentence
    :type references: list(str) / str
    :param hypothesis: a hypothesis sentence
    :type hypothesis: list(str) / str
    :param min_len: The minimum order of n-gram this function should extract.
    :type min_len: int
    :param max_len: The maximum order of n-gram this function should extract.
    :type max_len: int
    :param beta: the parameter to assign more importance to recall over precision
    :type beta: float
    :param ignore_whitespace: ignore whitespace characters in scoring
    :type ignore_whitespace: bool
    :return: the sentence level CHRF score.
    :rtype: float
    """
    return corpus_chrf([reference], [hypothesis], min_len, max_len, beta=beta)

def _preprocess(sent, ignore_whitespace):
    if type(sent) != str:
        # turn list of tokens into a string
        sent = ' '.join(sent)

    if ignore_whitespace:
        sent = re.sub(r'\s+', '', sent)
    return sent

def chrf_precision_recall_fscore_support(reference, hypothesis, n, beta=3.0,
                                          epsilon=1e-16):
    """
    This function computes the precision, recall and fscore from the ngram
    overlaps. It returns the `support` which is the true positive scoreself.

    By underspecifying the input type, the function will be agnostic as to how
    it computes the ngrams and simply take the whichever element in the list;
    it could be either token or character.

    :param reference: The reference sentence.
    :type reference: list
    :param hypothesis: The hypothesis sentence.
    :type hypothesis: list
    :param n: The ngram order.
    :type n: int
    :return: An NgramOverlapScores object that contains the TP, TPFP, TPFN, P, R and F scores.
    :rtype: NgramOverlapScores
    """
    ref_ngrams = Counter(ngrams(reference, order))
    hyp_ngrams = Counter(ngrams(hypothesis, order))

    # calculate the number of ngram matches
    overlap_ngrams = ref_ngrams & hyp_ngrams
    tp = sum(overlap_ngrams.values())  # True positives.
    tpfp = sum(hyp_ngrams.values())    # True positives + False positives.
    tffn = sum(ref_ngrams.values())    # True positives + False negatives.

    try:
        prec = tp / tpfp  # precision
        rec = tp / tffn   # recall
        factor = beta**2
        fscore = (1 + factor) * (prec * rec) / (factor * prec + rec)
    except ZeroDivisionError:
        prec = rec = fscore = epsilon
    return prec, rec, fscore, tp

def corpus_chrf(references, hypotheses, min_len=1, max_len=6, beta=3.0,
                ignore_whitespace=True):
    """
    Calculates the corpus level CHRF (Character n-gram F-score), it is the
    macro-averaged value of the sentence/segment level CHRF score.

    This implementation of CHRF only supports a single reference at the moment.

        >>> ref1 = str('It is a guide to action that ensures that the military '
        ...            'will forever heed Party commands').split()
        >>> ref2 = str('It is the guiding principle which guarantees the military '
        ...            'forces always being under the command of the Party').split()
        >>>
        >>> hyp1 = str('It is a guide to action which ensures that the military '
        ...            'always obeys the commands of the party').split()
        >>> hyp2 = str('It is to insure the troops forever hearing the activity '
        ...            'guidebook that party direct')
        >>> corpus_chrf([ref1, ref2, ref1, ref2], [hyp1, hyp2, hyp2, hyp1]) # doctest: +ELLIPSIS
        0.3910...

    :param references: a corpus of list of reference sentences, w.r.t. hypotheses
    :type references: list(list(str)) / list(str)
    :param hypotheses: a list of hypothesis sentences
    :type hypotheses: list(list(str)) / list(str)
    :param min_len: The minimum order of n-gram this function should extract.
    :type min_len: int
    :param max_len: The maximum order of n-gram this function should extract.
    :type max_len: int
    :param beta: the parameter to assign more importance to recall over precision
    :type beta: float
    :param ignore_whitespace: ignore whitespace characters in scoring
    :type ignore_whitespace: bool
    :return: the sentence level CHRF score.
    :rtype: float
    """

    assert len(references) == len(hypotheses), (
        "The number of hypotheses and their references should be the same")

    # Keep f-scores for each n-gram order separate
    ngram_fscores = defaultdict(lambda: list())

    # Iterate through each hypothesis and their corresponding references.
    for reference, hypothesis in zip(references, hypotheses):

        # preprocess both reference and hypothesis
        reference = _preprocess(reference, ignore_whitespace)
        hypothesis = _preprocess(hypothesis, ignore_whitespace)

        # Calculate f-scores for each sentence and for each n-gram order
        # separately.
        for n in range(min_len, max_len + 1):
            # Compute the precision, recall, fscore and support.
            prec, rec, fscore, tp = chrf_precision_recall_fscore_support(reference, hypothesis, n)
            ngram_fscores[n].append(fscore)

    num_sents = len(ngram_fscores[min_len])

    # This is not specified in the paper but the author's implementation
    # computes macro-averages both over n-gram lengths and sentences.

    # how many n-gram sizes
    num_ngram_sizes = len(ngram_fscores)

    # sum of f-scores over all sentences for each n-gram order
    total_scores = [sum(ngram_fscores[n]) for n in range(min_len, max_len + 1)]

    # macro-average over n-gram orders and over all sentences
    return (sum(total_scores) / num_ngram_sizes) / num_sents
<commit_msg>cleaner way to iterate through defaultdict(list)<commit_after># -*- coding: utf-8 -*-
# Natural Language Toolkit: ChrF score
#
# Copyright (C) 2001-2018 NLTK Project
# Authors: Maja Popovic
# Contributors: Liling Tan, Aleš Tamchyna (Memsource)
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

""" ChrF score implementation """
from __future__ import division
from collections import Counter, defaultdict
import re

from nltk.util import ngrams

def sentence_chrf(reference, hypothesis, min_len=1, max_len=6, beta=3.0,
                  ignore_whitespace=True):
    """
    Calculates the sentence level CHRF (Character n-gram F-score) described in
     - Maja Popovic. 2015. CHRF: Character n-gram F-score for Automatic MT Evaluation.
       In Proceedings of the 10th Workshop on Machine Translation.
       http://www.statmt.org/wmt15/pdf/WMT49.pdf
     - Maja Popovic. 2016. CHRF Deconstructed: β Parameters and n-gram Weights.
       In Proceedings of the 1st Conference on Machine Translation.
       http://www.statmt.org/wmt16/pdf/W16-2341.pdf

    This implementation of CHRF only supports a single reference at the moment.

    For details not reported in the paper, consult Maja Popovic's original
    implementation: https://github.com/m-popovic/chrF

    The code should output results equivalent to running CHRF++ with the
    following options: -nw 0 -b 3

    An example from the original BLEU paper
    http://www.aclweb.org/anthology/P02-1040.pdf

        >>> ref1 = str('It is a guide to action that ensures that the military '
        ...            'will forever heed Party commands').split()
        >>> hyp1 = str('It is a guide to action which ensures that the military '
        ...            'always obeys the commands of the party').split()
        >>> hyp2 = str('It is to insure the troops forever hearing the activity '
        ...            'guidebook that party direct').split()
        >>> sentence_chrf(ref1, hyp1) # doctest: +ELLIPSIS
        0.6349...
        >>> sentence_chrf(ref1, hyp2) # doctest: +ELLIPSIS
        0.3330...

    The infamous "the the the ... " example

        >>> ref = 'the cat is on the mat'.split()
        >>> hyp = 'the the the the the the the'.split()
        >>> sentence_chrf(ref, hyp)  # doctest: +ELLIPSIS
        0.1468...

    An example to show that this function allows users to use strings instead of
    tokens, i.e. list(str) as inputs.

        >>> ref1 = str('It is a guide to action that ensures that the military '
        ...            'will forever heed Party commands')
        >>> hyp1 = str('It is a guide to action which ensures that the military '
        ...            'always obeys the commands of the party')
        >>> sentence_chrf(ref1, hyp1) # doctest: +ELLIPSIS
        0.6349...
        >>> type(ref1) == type(hyp1) == str
        True
        >>> sentence_chrf(ref1.split(), hyp1.split()) # doctest: +ELLIPSIS
        0.6349...

    To skip the unigrams and only use 2- to 3-grams:

        >>> sentence_chrf(ref1, hyp1, min_len=2, max_len=3) # doctest: +ELLIPSIS
        0.6617...

    :param references: reference sentence
    :type references: list(str) / str
    :param hypothesis: a hypothesis sentence
    :type hypothesis: list(str) / str
    :param min_len: The minimum order of n-gram this function should extract.
    :type min_len: int
    :param max_len: The maximum order of n-gram this function should extract.
    :type max_len: int
    :param beta: the parameter to assign more importance to recall over precision
    :type beta: float
    :param ignore_whitespace: ignore whitespace characters in scoring
    :type ignore_whitespace: bool
    :return: the sentence level CHRF score.
    :rtype: float
    """
    return corpus_chrf([reference], [hypothesis], min_len, max_len, beta=beta)

def _preprocess(sent, ignore_whitespace):
    if type(sent) != str:
        # turn list of tokens into a string
        sent = ' '.join(sent)

    if ignore_whitespace:
        sent = re.sub(r'\s+', '', sent)
    return sent

def chrf_precision_recall_fscore_support(reference, hypothesis, n, beta=3.0,
                                          epsilon=1e-16):
    """
    This function computes the precision, recall and fscore from the ngram
    overlaps. It returns the `support` which is the true positive scoreself.

    By underspecifying the input type, the function will be agnostic as to how
    it computes the ngrams and simply take the whichever element in the list;
    it could be either token or character.

    :param reference: The reference sentence.
    :type reference: list
    :param hypothesis: The hypothesis sentence.
    :type hypothesis: list
    :param n: The ngram order.
    :type n: int
    :return: An NgramOverlapScores object that contains the TP, TPFP, TPFN, P, R and F scores.
    :rtype: NgramOverlapScores
    """
    ref_ngrams = Counter(ngrams(reference, order))
    hyp_ngrams = Counter(ngrams(hypothesis, order))

    # calculate the number of ngram matches
    overlap_ngrams = ref_ngrams & hyp_ngrams
    tp = sum(overlap_ngrams.values())  # True positives.
    tpfp = sum(hyp_ngrams.values())    # True positives + False positives.
    tffn = sum(ref_ngrams.values())    # True positives + False negatives.

    try:
        prec = tp / tpfp  # precision
        rec = tp / tffn   # recall
        factor = beta**2
        fscore = (1 + factor) * (prec * rec) / (factor * prec + rec)
    except ZeroDivisionError:
        prec = rec = fscore = epsilon
    return prec, rec, fscore, tp

def corpus_chrf(references, hypotheses, min_len=1, max_len=6, beta=3.0,
                ignore_whitespace=True):
    """
    Calculates the corpus level CHRF (Character n-gram F-score), it is the
    macro-averaged value of the sentence/segment level CHRF score.

    This implementation of CHRF only supports a single reference at the moment.

        >>> ref1 = str('It is a guide to action that ensures that the military '
        ...            'will forever heed Party commands').split()
        >>> ref2 = str('It is the guiding principle which guarantees the military '
        ...            'forces always being under the command of the Party').split()
        >>>
        >>> hyp1 = str('It is a guide to action which ensures that the military '
        ...            'always obeys the commands of the party').split()
        >>> hyp2 = str('It is to insure the troops forever hearing the activity '
        ...            'guidebook that party direct')
        >>> corpus_chrf([ref1, ref2, ref1, ref2], [hyp1, hyp2, hyp2, hyp1]) # doctest: +ELLIPSIS
        0.3910...

    :param references: a corpus of list of reference sentences, w.r.t. hypotheses
    :type references: list(list(str)) / list(str)
    :param hypotheses: a list of hypothesis sentences
    :type hypotheses: list(list(str)) / list(str)
    :param min_len: The minimum order of n-gram this function should extract.
    :type min_len: int
    :param max_len: The maximum order of n-gram this function should extract.
    :type max_len: int
    :param beta: the parameter to assign more importance to recall over precision
    :type beta: float
    :param ignore_whitespace: ignore whitespace characters in scoring
    :type ignore_whitespace: bool
    :return: the sentence level CHRF score.
    :rtype: float
    """

    assert len(references) == len(hypotheses), (
        "The number of hypotheses and their references should be the same")

    # Keep f-scores for each n-gram order separate
    ngram_fscores = defaultdict(lambda: list())

    # Iterate through each hypothesis and their corresponding references.
    for reference, hypothesis in zip(references, hypotheses):

        # preprocess both reference and hypothesis
        reference = _preprocess(reference, ignore_whitespace)
        hypothesis = _preprocess(hypothesis, ignore_whitespace)

        # Calculate f-scores for each sentence and for each n-gram order
        # separately.
        for n in range(min_len, max_len + 1):
            # Compute the precision, recall, fscore and support.
            prec, rec, fscore, tp = chrf_precision_recall_fscore_support(reference, hypothesis, n)
            ngram_fscores[n].append(fscore)

    num_sents = len(ngram_fscores[min_len])

    # This is not specified in the paper but the author's implementation
    # computes macro-averages both over n-gram lengths and sentences.

    # how many n-gram sizes
    num_ngram_sizes = len(ngram_fscores)

    # sum of f-scores over all sentences for each n-gram order
    total_scores = [sum(fscores) for n, fscores in ngram_fscores.items()]

    # macro-average over n-gram orders and over all sentences
    return (sum(total_scores) / num_ngram_sizes) / num_sents
<|endoftext|>


<commit_before>"""
Client library for interacting with a Higgla server over HTTP/JSON
"""

try:
    import json
except ImportError:
    import simplejson as json

import socket

class Session:
    """
    A connection to a Higgla server configured to talk to a given base.
    """
    def __init__ (self, host, port, base):
        """
        Create a new session

        :param host: The host name of the server running the Higgla service
        :param port: The port on which the remote Higgla service is available.
            The default port for the Higgla server is 4567.
        :param base: Name of the base this session should query and store boxes
           in
        """
        self._host = host
        self._port = port
        self._base = base
        self._http = HTTPTransactionFactory(
                           hostname=host, port=port, body_parser=json.load)

    def prepare_box(self, id, revision, *index, **kwargs):
        """
        Prepare a new empty box for storage in the base. The return value
        is a :const:`dict` type

        :param id: The unique string id for the box
        :param revision: The revision number last seen for this box. Use 0 for
            creating a new box. This version number `must` match the version
            number that is current in the Higgla base; otherwise :meth:`store`
            will raise a :class:VersionConflict when you try and store the box
        """
        if not isinstance(id, (str,unicode)) :
            raise TypeError("Box id must be string or unicode, found %s" % type(id))
        if not isinstance(revision, (int,long)):
            raise TypeError("Box revision must be int or long, found %s" % type(revision))
        box = {}
        box["__id__"] = id
        box["__rev__"] = revision
        box["__index__"] = index
        box.update(kwargs)
        return box

    def prepare_query(self, **kwargs):
        q = Query(self._base)

        if kwargs:
            q.add_template(kwargs)
        return q

    def send_query(self, query):
        return self.send("POST", "/actor/query/", query._q)

    def get(self, ids):
        if not isinstance(ids, list):
            raise TypeError(
                       "Ids must be a list, found %s" % type(dict))
        msg = {"__ids__" : ids, "__base__" : self._base}
        return self.send("POST", "/actor/get/", msg)

    def store(self, boxes):
        if not isinstance(boxes, list):
            raise TypeError(
                       "Expected list, found %s" % type(boxes))
        msg = {"__store__" : boxes, "__base__" : self._base}
        return self.send("POST", "/actor/store/", msg)

    def send(self, method, url, msg):
        """
        Expert: Send a free form message to the Higgla server

        :param method:
        :param uri:
        :param msg:
        :returns:
        """
        conn = self._http.create()
        conn.send_request(method, url, {} ,msg)
        headers, body = conn.read_response()
        self._check_error(body)
        return body

    def _check_error(self, body):
        if body.has_key("error"):
            error = body["error"]
            if error == "conflict":
                raise VersionConflict(body["__id__"], box["__rev__"])
            raise HigglaException(error)

class Query:
    def __init__ (self, base):
        self._q = {}
        self._q["__base__"] = base
        self._q["__query__"] = []

    def add_template(self, template):
        if not isinstance(template, dict):
            raise TypeError(
                       "Query template must be a dict, found %s" % type(dict))
        self._q["__query__"].append(template)

class SocketFactory:
    def __init__ (self, hostname, port):
        self._hostname = hostname
        self._port = port
        self._socket = None
        self._lastaddr = None

    def connect(self):
        """
        Return a newly created socket.Socket instance. The last known
        working address will be cached in order to not incur the overhead
        of name resolution on each call. If the cached address fails to
        open the host and port will be resolved again.
        """
        # Try and reuse the old address
        s = None
        if self._lastaddr:
            af, socktype, proto, canonname, sa = self._lastaddr
            try:
                s = socket.socket(af, socktype, proto)
            except socket.error, msg:
                s = None
            try:
                s.connect(sa)
            except socket.error, msg:
                s.close()
                s = None
        if s:
            return s

        # The old address did work, try resolving the address again
        for res in socket.getaddrinfo(
              self._hostname, self._port, socket.AF_UNSPEC, socket.SOCK_STREAM):
            af, socktype, proto, canonname, sa = res
            try:
                s = socket.socket(af, socktype, proto)
            except socket.error, msg:
                s = None
                continue
            try:
                s.connect(sa)
            except socket.error, msg:
                s.close()
                s = None
                continue

            # We are good, cache this address
            self._lastaddr = res
            break
        return s

class HTTPProtocolError (Exception):
    def __init__ (self, msg):
        Exception.__init__(self, msg)

class HigglaException (Exception):
    """
    Represents a server side error
    """
    def __init__ (self, msg):
        Exception.__init__(self, msg)

class VersionConflict (HigglaException):
    def __init__ (self, box_id, expected_revision):
        HigglaException.__init__(
         self, "Expected revision %s of box '%s'" % (expected_revision, box_id))
        self._box_id = box_id
        self._expected_revision = expected_revision

    def get_box_id(self) :
        return self._box_id
    id = property(get_box_id, doc="")

    def get_expected_revision(self):
        return self._expected_revision
    expected_revision = property(get_expected_revision)

class HTTPTransaction:

    def __init__ (self, sock, body_parser=None):
        self._socket = sock
        self._file = None
        self._body_parser = body_parser

    def send_request(self, method, url, headers, body=None):
        msg = "%s %s %s\r\n" % (method, url, "HTTP/1.0")
        for k, v in headers.iteritems():
            msg += "%s: %s\r\n" % (k, v)
        msg += "\r\n"
        if body:
            if isinstance(body, (dict,list)):
                body = json.dumps(body)
            else:
                body = str(body)
            msg += body
        self._socket.sendall(msg)

    def read_response(self, bufsize=1024):
        self._file = self._socket.makefile("r", bufsize)
        self._socket.close()
        self._socket = None

        proto, status, reason = self._file.readline().split()

        if not proto in ("HTTP/1.0", "HTTP/1.1"):
            raise HTTPProtocolError("Illegal protocol declaration %s" % proto)

        status = int(status)
        if status < 100 or status > 999:
            raise HTTPProtocolError("Illegal status code %s" % status)

        headers = {}
        while True:
            line = self._file.readline()
            if line == "\r\n" : break
            k, v = line.split(":")
            headers[k] = v.strip()

        # caller will own self._file
        body_stream = self._file
        self._file = None

        if self._body_parser:
            try:
                body = self._body_parser(body_stream)
            finally:
                body_stream.close()
            return headers, body
        else:
            return headers, body_stream

    def close(self):
        if self._socket :
            self._socket.close()
            self._socket = None
        if self._file:
            self._file.close()
            self._file = None

class HTTPTransactionFactory:
    def __init__ (self, socket_factory=None,
                  hostname=None, port=None, body_parser=None):
        if socket_factory:
            self._sockets = socket_factory
        elif hostname and port:
            self._sockets = SocketFactory(hostname, port)
        else:
            raise TypeError("Illegal arguments")

        self._body_parser = body_parser

    def create(self):
        s = self._sockets.connect()
        return HTTPTransaction(s, body_parser=self._body_parser)

if __name__ == "__main__":
    session = Session("localhost", 4567, "mybase")

    print "LOOKUP mydoc1 AND mydoc2 RESULTS"
    print str(session.get(["mydoc1", "mydoc2"]))
    print ""

    print "STORE RESULTS"
    box = session.prepare_box("mke", 4, "firstname",
                              firstname="Mikkel", lastname="Kamstrup")
    box["address"] = "57 Mount Pleasant Street"
    try:
        print str(session.store([box]))
    except HigglaException, e:
        print "Error: %s" % e
    print ""

    print "QUERY RESULTS"
    q = session.prepare_query(firstname="mikkel")
    print str(session.send_query(q))
    print ""
<commit_msg>Catch HigglaExceptions in the sample main method of higgla/client.py<commit_after>"""
Client library for interacting with a Higgla server over HTTP/JSON
"""

try:
    import json
except ImportError:
    import simplejson as json

import socket

class Session:
    """
    A connection to a Higgla server configured to talk to a given base.
    """
    def __init__ (self, host, port, base):
        """
        Create a new session

        :param host: The host name of the server running the Higgla service
        :param port: The port on which the remote Higgla service is available.
            The default port for the Higgla server is 4567.
        :param base: Name of the base this session should query and store boxes
           in
        """
        self._host = host
        self._port = port
        self._base = base
        self._http = HTTPTransactionFactory(
                           hostname=host, port=port, body_parser=json.load)

    def prepare_box(self, id, revision, *index, **kwargs):
        """
        Prepare a new empty box for storage in the base. The return value
        is a :const:`dict` type

        :param id: The unique string id for the box
        :param revision: The revision number last seen for this box. Use 0 for
            creating a new box. This version number `must` match the version
            number that is current in the Higgla base; otherwise :meth:`store`
            will raise a :class:VersionConflict when you try and store the box
        """
        if not isinstance(id, (str,unicode)) :
            raise TypeError("Box id must be string or unicode, found %s" % type(id))
        if not isinstance(revision, (int,long)):
            raise TypeError("Box revision must be int or long, found %s" % type(revision))
        box = {}
        box["__id__"] = id
        box["__rev__"] = revision
        box["__index__"] = index
        box.update(kwargs)
        return box

    def prepare_query(self, **kwargs):
        q = Query(self._base)

        if kwargs:
            q.add_template(kwargs)
        return q

    def send_query(self, query):
        return self.send("POST", "/actor/query/", query._q)

    def get(self, ids):
        if not isinstance(ids, list):
            raise TypeError(
                       "Ids must be a list, found %s" % type(dict))
        msg = {"__ids__" : ids, "__base__" : self._base}
        return self.send("POST", "/actor/get/", msg)

    def store(self, boxes):
        if not isinstance(boxes, list):
            raise TypeError(
                       "Expected list, found %s" % type(boxes))
        msg = {"__store__" : boxes, "__base__" : self._base}
        return self.send("POST", "/actor/store/", msg)

    def send(self, method, url, msg):
        """
        Expert: Send a free form message to the Higgla server

        :param method:
        :param uri:
        :param msg:
        :returns:
        """
        conn = self._http.create()
        conn.send_request(method, url, {} ,msg)
        headers, body = conn.read_response()
        self._check_error(body)
        return body

    def _check_error(self, body):
        if body.has_key("error"):
            error = body["error"]
            if error == "conflict":
                raise VersionConflict(body["__id__"], box["__rev__"])
            raise HigglaException(error)

class Query:
    def __init__ (self, base):
        self._q = {}
        self._q["__base__"] = base
        self._q["__query__"] = []

    def add_template(self, template):
        if not isinstance(template, dict):
            raise TypeError(
                       "Query template must be a dict, found %s" % type(dict))
        self._q["__query__"].append(template)

class SocketFactory:
    def __init__ (self, hostname, port):
        self._hostname = hostname
        self._port = port
        self._socket = None
        self._lastaddr = None

    def connect(self):
        """
        Return a newly created socket.Socket instance. The last known
        working address will be cached in order to not incur the overhead
        of name resolution on each call. If the cached address fails to
        open the host and port will be resolved again.
        """
        # Try and reuse the old address
        s = None
        if self._lastaddr:
            af, socktype, proto, canonname, sa = self._lastaddr
            try:
                s = socket.socket(af, socktype, proto)
            except socket.error, msg:
                s = None
            try:
                s.connect(sa)
            except socket.error, msg:
                s.close()
                s = None
        if s:
            return s

        # The old address did work, try resolving the address again
        for res in socket.getaddrinfo(
              self._hostname, self._port, socket.AF_UNSPEC, socket.SOCK_STREAM):
            af, socktype, proto, canonname, sa = res
            try:
                s = socket.socket(af, socktype, proto)
            except socket.error, msg:
                s = None
                continue
            try:
                s.connect(sa)
            except socket.error, msg:
                s.close()
                s = None
                continue

            # We are good, cache this address
            self._lastaddr = res
            break
        return s

class HTTPProtocolError (Exception):
    def __init__ (self, msg):
        Exception.__init__(self, msg)

class HigglaException (Exception):
    """
    Represents a server side error
    """
    def __init__ (self, msg):
        Exception.__init__(self, msg)

class VersionConflict (HigglaException):
    def __init__ (self, box_id, expected_revision):
        HigglaException.__init__(
         self, "Expected revision %s of box '%s'" % (expected_revision, box_id))
        self._box_id = box_id
        self._expected_revision = expected_revision

    def get_box_id(self) :
        return self._box_id
    id = property(get_box_id, doc="")

    def get_expected_revision(self):
        return self._expected_revision
    expected_revision = property(get_expected_revision)

class HTTPTransaction:

    def __init__ (self, sock, body_parser=None):
        self._socket = sock
        self._file = None
        self._body_parser = body_parser

    def send_request(self, method, url, headers, body=None):
        msg = "%s %s %s\r\n" % (method, url, "HTTP/1.0")
        for k, v in headers.iteritems():
            msg += "%s: %s\r\n" % (k, v)
        msg += "\r\n"
        if body:
            if isinstance(body, (dict,list)):
                body = json.dumps(body)
            else:
                body = str(body)
            msg += body
        self._socket.sendall(msg)

    def read_response(self, bufsize=1024):
        self._file = self._socket.makefile("r", bufsize)
        self._socket.close()
        self._socket = None

        proto, status, reason = self._file.readline().split()

        if not proto in ("HTTP/1.0", "HTTP/1.1"):
            raise HTTPProtocolError("Illegal protocol declaration %s" % proto)

        status = int(status)
        if status < 100 or status > 999:
            raise HTTPProtocolError("Illegal status code %s" % status)

        headers = {}
        while True:
            line = self._file.readline()
            if line == "\r\n" : break
            k, v = line.split(":")
            headers[k] = v.strip()

        # caller will own self._file
        body_stream = self._file
        self._file = None

        if self._body_parser:
            try:
                body = self._body_parser(body_stream)
            finally:
                body_stream.close()
            return headers, body
        else:
            return headers, body_stream

    def close(self):
        if self._socket :
            self._socket.close()
            self._socket = None
        if self._file:
            self._file.close()
            self._file = None

class HTTPTransactionFactory:
    def __init__ (self, socket_factory=None,
                  hostname=None, port=None, body_parser=None):
        if socket_factory:
            self._sockets = socket_factory
        elif hostname and port:
            self._sockets = SocketFactory(hostname, port)
        else:
            raise TypeError("Illegal arguments")

        self._body_parser = body_parser

    def create(self):
        s = self._sockets.connect()
        return HTTPTransaction(s, body_parser=self._body_parser)

if __name__ == "__main__":
    session = Session("localhost", 4567, "mybase")

    print "LOOKUP mydoc1 AND mydoc2 RESULTS"
    try:
        print str(session.get(["mydoc1", "mydoc2"]))
    except HigglaException, e:
        print e
    print ""

    print "STORE RESULTS"
    box = session.prepare_box("mke", 0, "firstname",
                              firstname="Mikkel", lastname="Kamstrup")
    box["address"] = "57 Mount Pleasant Street"
    try:
        print str(session.store([box]))
    except HigglaException, e:
        print "Error: %s" % e
    print ""

    print "QUERY RESULTS"
    q = session.prepare_query(firstname="mikkel")
    print str(session.send_query(q))
    print ""
<|endoftext|>