<commit_before>#!/usr/bin/env python

# Copyright (c) 2013, Carnegie Mellon University
# All rights reserved.
# Authors: Michael Koval <mkoval@cs.cmu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# - Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# - Neither the name of Carnegie Mellon University nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import numpy, openravepy
from manipulator import Manipulator
from prpy.clone import Clone, Cloned
from .. import util
from .. import exceptions

class WAM(Manipulator):
    def __init__(self, sim, owd_namespace,
                 iktype=openravepy.IkParameterization.Type.Transform6D):
        Manipulator.__init__(self)

        self.simulated = sim
        self.controller = self.GetRobot().AttachController(name=self.GetName(),
            args='OWDController {0:s} {1:s}'.format('prpy', owd_namespace),
            dof_indices=self.GetArmIndices(), affine_dofs=0, simulated=sim)

        # Load the IK database.
        robot = self.GetRobot()
        if iktype is not None:
            with robot:
                self.SetActive()
                self.ikmodel = openravepy.databases.inversekinematics.InverseKinematicsModel(robot, iktype=iktype)
                if not self.ikmodel.load():
                    self.ikmodel.autogenerate()

        # Enable servo motions in simulation mode.
        if sim:
            from prpy.simulation import ServoSimulator
            self.servo_simulator = ServoSimulator(self, rate=20, watchdog_timeout=0.1)

    def CloneBindings(self, parent):
        self.__init__(True, None)

    def SetStiffness(manipulator, stiffness):
        """
        Set the WAM's stiffness. This enables or disables gravity compensation.
        Values between 0 and 1 are experimental.
        @param stiffness value between 0.0 and 1.0
        """
        if not (0 <= stiffness <= 1):
            raise Exception('Stiffness must in the range [0, 1]; got %f.' % stiffness)

        if not manipulator.simulated:
            manipulator.controller.SendCommand('SetStiffness {0:f}'.format(stiffness))

    def Servo(manipulator, velocities):
        """
        Servo with an instantaneous vector joint velocities.
        @param velocities instantaneous joint velocities in radians per second
        """
        num_dof = len(manipulator.GetArmIndices())
        if len(velocities) != num_dof:
            raise ValueError('Incorrect number of joint velocities. Expected {0:d}; got {0:d}.'.format(
                             num_dof, len(velocities)))

        if not manipulator.simulated:
            manipulator.controller.SendCommand('Servo ' + ' '.join([ str(qdot) for qdot in velocities ]))
        else:
            manipulator.controller.Reset(0)
            manipulator.servo_simulator.SetVelocity(velocities)

    def ServoTo(manipulator, target, duration, timeStep = 0.05, collisionChecking= True):
        """
        Servo's the WAM to the target taking the duration passed to it
        @param target desired joint angles
        @param duration duration in seconds
        @param timestep period of the control loop
        @param collisionchecking check collisions in the simulation environment
        """
        steps = int(math.ceil(duration/timeStep))
        original_dofs = manipulator.GetRobot().GetDOFValues(manipulator.GetArmIndices())
        velocity = numpy.array(target-manipulator.GetRobot().GetDOFValues(manipulator.GetArmIndices()))
        velocities = v/steps#[v/steps for v in velocity]
        inCollision = False
        if collisionChecking==True:
            inCollision = manipulator.CollisionCheck(target)
        if inCollision == False:
            for i in range(1,steps):
                manipulator.Servo(velocities)
                time.sleep(timeStep)
            manipulator.Servo([0] * len(manipulator.GetArmIndices()))
            new_dofs = manipulator.GetRobot().GetDOFValues(manipulator.GetArmIndices())
            return True
        return False

    def GetVelocityLimits(self, openrave=True, owd=True):
        """
        Get the OpenRAVE and OWD joint velocity limits. If they
        do not match, up to some epsilon, the minimum value is taken,
        and an error is printed.
        @param openrave flag to set the OpenRAVE velocity limits
        @param owd flag to set the OWD velocity limits
        """
        # Update the OpenRAVE limits.
        if openrave:
            or_velocity_limits = Manipulator.GetVelocityLimits(self)
            if self.simulated or not owd:
                return or_velocity_limits

        # Update the OWD limits.
        if owd and not self.simulated:
            args  = [ 'GetSpeed' ]
            args_str = ' '.join(args)
            owd_speed_limits_all = self.controller.SendCommand(args_str)
            #if we get nothing back, e.g. if the arm isn't running, return openrave lims
            if owd_speed_limits_all is None:
                return or_velocity_limits

            owd_speed_limits = map(float, owd_speed_limits_all.split(','));
            #first 7 numbers are velocity limits
            owd_velocity_limits = numpy.array(owd_speed_limits[0:len(self.GetIndices())])

            diff_arr = numpy.subtract(or_velocity_limits, owd_velocity_limits)
            max_diff = max(abs(diff_arr))

            if (max_diff > 0.01):
                print('GetVelocityLimits Error: openrave and owd limits very different')
                print('\tOpenrave limits:\t' + str(or_velocity_limits))
                print('\tOWD limits:\t\t' + str(owd_velocity_limits))

            return numpy.minimum(or_velocity_limits, owd_velocity_limits)

        return or_velocity_limits

    def SetVelocityLimits(self, velocity_limits, min_accel_time,
                          openrave=True, owd=True):
        """
        Change the OpenRAVE and OWD joint velocity limits. Joint velocities
        that exceed these limits will trigger a velocity fault.
        @param velocity_limits vector of joint velocity limits in radians per second
        @param min_accel_time minimum acceleration time used to compute acceleration limits
        @param openrave flag to set the OpenRAVE velocity limits
        @param owd flag to set the OWD velocity limits
        """
        # Update the OpenRAVE limits.
        if openrave:
            Manipulator.SetVelocityLimits(self, velocity_limits, min_accel_time)

        # Update the OWD limits.
        if owd and not self.simulated:
            args  = [ 'SetSpeed' ]
            args += [ str(min_accel_time) ]
            args += [ str(velocity) for velocity in velocity_limits ]
            args_str = ' '.join(args)
            self.controller.SendCommand(args_str)

    def GetTrajectoryStatus(manipulator):
        """
        Gets the status of the current (or previous) trajectory executed by the
        controller.
        @return status of the current (or previous) trajectory executed
        """
        if not manipulator.simulated:
            return manipulator.controller.SendCommand('GetStatus')
        else:
            if manipulator.controller.IsDone():
                return 'done'
            else:
                return 'active'

    def ClearTrajectoryStatus(manipulator):
        """
        Clears the current trajectory execution status.
        """
        if not manipulator.simulated:
            manipulator.controller.SendCommand('ClearStatus')

    def MoveUntilTouch(manipulator, direction, distance, max_distance=float('+inf'),
                       max_force=5.0, max_torque=None, ignore_collisions=None, **kw_args):
        """
        Execute a straight move-until-touch action. This action stops when a
        sufficient force is is felt or the manipulator moves the maximum
        distance. The motion is considered successful if the end-effector moves
        at least distance.
        @param direction unit vector for the direction of motion in the world frame
        @param distance minimum distance in meters
        @param max_distance maximum distance in meters
        @param max_force maximum force in Newtons
        @param torque maximum torque in Newton-Meters
        @param execute optionally execute the trajectory
        @param **kw_args planner parameters
        @return felt_force flag indicating whether we felt a force.
        """
        if max_torque is None:
            max_torque = numpy.array([100.0, 100.0, 100.0 ])

        ignore_col_obj_oldstate = []
        if ignore_collisions is None:
            ignore_collisions = []

        for ignore_col_with in ignore_collisions:
            ignore_col_obj_oldstate.append(ignore_col_with.IsEnabled())
            ignore_col_with.Enable(False)

        with manipulator.GetRobot().GetEnv():
            manipulator.GetRobot().GetController().SimulationStep(0)

            # Compute the expected force direction in the hand frame.
            direction = numpy.array(direction)
            hand_pose = manipulator.GetEndEffectorTransform()
            force_direction = numpy.dot(hand_pose[0:3, 0:3].T, -direction)

            with manipulator.GetRobot():
                old_active_manipulator = manipulator.GetRobot().GetActiveManipulator()
                manipulator.SetActive()
                traj = manipulator.PlanToEndEffectorOffset(direction, distance, max_distance=max_distance,
                                                           execute=False, **kw_args)
                traj = manipulator.GetRobot().BlendTrajectory(traj)
                traj = manipulator.GetRobot().RetimeTrajectory(traj, stop_on_ft=True, force_direction=force_direction,
                                                               force_magnitude=max_force, torque=max_torque)

	collided_with_obj = False
        try:
            if not manipulator.simulated:
                manipulator.hand.TareForceTorqueSensor()
                manipulator.GetRobot().ExecuteTrajectory(traj, execute=True, retime=False, blend=False)
                for (ignore_col_with, oldstate) in zip(ignore_collisions, ignore_col_obj_oldstate):
                    ignore_col_with.Enable(oldstate)
            else:
                traj_duration = traj.GetDuration()
                delta_t = 0.01

                traj_config_spec = traj.GetConfigurationSpecification()
                traj_angle_group = traj_config_spec.GetGroupFromName('joint_values')
                path_config_spec = openravepy.ConfigurationSpecification()
#                path_config_spec.AddDeltaTimeGroup()
                path_config_spec.AddGroup(traj_angle_group.name, traj_angle_group.dof, '')
#                path_config_spec.AddDerivativeGroups(1, False);
#                path_config_spec.AddDerivativeGroups(2, False);
#                path_config_spec.AddGroup('owd_blend_radius', 1, 'next')
                path_config_spec.ResetGroupOffsets()

                new_traj = openravepy.RaveCreateTrajectory(manipulator.GetRobot().GetEnv(), '')
                new_traj.Init(path_config_spec)

                for (ignore_col_with, oldstate) in zip(ignore_collisions, ignore_col_obj_oldstate):
                    ignore_col_with.Enable(oldstate)

                with manipulator.GetRobot():
                    manipulator.SetActive()
                    waypoint_ind = 0
                    for t in numpy.arange(0, traj_duration, delta_t):
                        traj_sample = traj.Sample(t)

                        waypoint = traj_config_spec.ExtractJointValues(traj_sample, manipulator.GetRobot(), manipulator.GetArmIndices())
                        manipulator.SetDOFValues(waypoint)
                        if manipulator.GetRobot().GetEnv().CheckCollision(manipulator.GetRobot()):
                            collided_with_obj = True
                            break
                        else:
                            #waypoint = numpy.append(waypoint,t)
                            new_traj.Insert(int(waypoint_ind), waypoint, path_config_spec)
                            waypoint_ind += 1

                    new_traj = manipulator.GetRobot().BlendTrajectory(new_traj)
                    new_traj = manipulator.GetRobot().RetimeTrajectory(new_traj)
                manipulator.GetRobot().ExecuteTrajectory(new_traj, execute = True, retime=False, blend=False)

            return collided_with_obj
        # Trajectory is aborted by OWD because we felt a force.
        except exceptions.TrajectoryAborted:
            return True
<commit_msg>fixed moveuntiltouch for simulation<commit_after>#!/usr/bin/env python

# Copyright (c) 2013, Carnegie Mellon University
# All rights reserved.
# Authors: Michael Koval <mkoval@cs.cmu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# - Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# - Neither the name of Carnegie Mellon University nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import numpy, openravepy
from manipulator import Manipulator
from prpy.clone import Clone, Cloned
from .. import util
from .. import exceptions

class WAM(Manipulator):
    def __init__(self, sim, owd_namespace,
                 iktype=openravepy.IkParameterization.Type.Transform6D):
        Manipulator.__init__(self)

        self.simulated = sim
        self.controller = self.GetRobot().AttachController(name=self.GetName(),
            args='OWDController {0:s} {1:s}'.format('prpy', owd_namespace),
            dof_indices=self.GetArmIndices(), affine_dofs=0, simulated=sim)

        # Load the IK database.
        robot = self.GetRobot()
        if iktype is not None:
            with robot:
                self.SetActive()
                self.ikmodel = openravepy.databases.inversekinematics.InverseKinematicsModel(robot, iktype=iktype)
                if not self.ikmodel.load():
                    self.ikmodel.autogenerate()

        # Enable servo motions in simulation mode.
        if sim:
            from prpy.simulation import ServoSimulator
            self.servo_simulator = ServoSimulator(self, rate=20, watchdog_timeout=0.1)

    def CloneBindings(self, parent):
        self.__init__(True, None)

    def SetStiffness(manipulator, stiffness):
        """
        Set the WAM's stiffness. This enables or disables gravity compensation.
        Values between 0 and 1 are experimental.
        @param stiffness value between 0.0 and 1.0
        """
        if not (0 <= stiffness <= 1):
            raise Exception('Stiffness must in the range [0, 1]; got %f.' % stiffness)

        if not manipulator.simulated:
            manipulator.controller.SendCommand('SetStiffness {0:f}'.format(stiffness))

    def Servo(manipulator, velocities):
        """
        Servo with an instantaneous vector joint velocities.
        @param velocities instantaneous joint velocities in radians per second
        """
        num_dof = len(manipulator.GetArmIndices())
        if len(velocities) != num_dof:
            raise ValueError('Incorrect number of joint velocities. Expected {0:d}; got {0:d}.'.format(
                             num_dof, len(velocities)))

        if not manipulator.simulated:
            manipulator.controller.SendCommand('Servo ' + ' '.join([ str(qdot) for qdot in velocities ]))
        else:
            manipulator.controller.Reset(0)
            manipulator.servo_simulator.SetVelocity(velocities)

    def ServoTo(manipulator, target, duration, timeStep = 0.05, collisionChecking= True):
        """
        Servo's the WAM to the target taking the duration passed to it
        @param target desired joint angles
        @param duration duration in seconds
        @param timestep period of the control loop
        @param collisionchecking check collisions in the simulation environment
        """
        steps = int(math.ceil(duration/timeStep))
        original_dofs = manipulator.GetRobot().GetDOFValues(manipulator.GetArmIndices())
        velocity = numpy.array(target-manipulator.GetRobot().GetDOFValues(manipulator.GetArmIndices()))
        velocities = v/steps#[v/steps for v in velocity]
        inCollision = False
        if collisionChecking==True:
            inCollision = manipulator.CollisionCheck(target)
        if inCollision == False:
            for i in range(1,steps):
                manipulator.Servo(velocities)
                time.sleep(timeStep)
            manipulator.Servo([0] * len(manipulator.GetArmIndices()))
            new_dofs = manipulator.GetRobot().GetDOFValues(manipulator.GetArmIndices())
            return True
        return False

    def GetVelocityLimits(self, openrave=True, owd=True):
        """
        Get the OpenRAVE and OWD joint velocity limits. If they
        do not match, up to some epsilon, the minimum value is taken,
        and an error is printed.
        @param openrave flag to set the OpenRAVE velocity limits
        @param owd flag to set the OWD velocity limits
        """
        # Update the OpenRAVE limits.
        if openrave:
            or_velocity_limits = Manipulator.GetVelocityLimits(self)
            if self.simulated or not owd:
                return or_velocity_limits

        # Update the OWD limits.
        if owd and not self.simulated:
            args  = [ 'GetSpeed' ]
            args_str = ' '.join(args)
            owd_speed_limits_all = self.controller.SendCommand(args_str)
            #if we get nothing back, e.g. if the arm isn't running, return openrave lims
            if owd_speed_limits_all is None:
                return or_velocity_limits

            owd_speed_limits = map(float, owd_speed_limits_all.split(','));
            #first 7 numbers are velocity limits
            owd_velocity_limits = numpy.array(owd_speed_limits[0:len(self.GetIndices())])

            diff_arr = numpy.subtract(or_velocity_limits, owd_velocity_limits)
            max_diff = max(abs(diff_arr))

            if (max_diff > 0.01):
                print('GetVelocityLimits Error: openrave and owd limits very different')
                print('\tOpenrave limits:\t' + str(or_velocity_limits))
                print('\tOWD limits:\t\t' + str(owd_velocity_limits))

            return numpy.minimum(or_velocity_limits, owd_velocity_limits)

        return or_velocity_limits

    def SetVelocityLimits(self, velocity_limits, min_accel_time,
                          openrave=True, owd=True):
        """
        Change the OpenRAVE and OWD joint velocity limits. Joint velocities
        that exceed these limits will trigger a velocity fault.
        @param velocity_limits vector of joint velocity limits in radians per second
        @param min_accel_time minimum acceleration time used to compute acceleration limits
        @param openrave flag to set the OpenRAVE velocity limits
        @param owd flag to set the OWD velocity limits
        """
        # Update the OpenRAVE limits.
        if openrave:
            Manipulator.SetVelocityLimits(self, velocity_limits, min_accel_time)

        # Update the OWD limits.
        if owd and not self.simulated:
            args  = [ 'SetSpeed' ]
            args += [ str(min_accel_time) ]
            args += [ str(velocity) for velocity in velocity_limits ]
            args_str = ' '.join(args)
            self.controller.SendCommand(args_str)

    def GetTrajectoryStatus(manipulator):
        """
        Gets the status of the current (or previous) trajectory executed by the
        controller.
        @return status of the current (or previous) trajectory executed
        """
        if not manipulator.simulated:
            return manipulator.controller.SendCommand('GetStatus')
        else:
            if manipulator.controller.IsDone():
                return 'done'
            else:
                return 'active'

    def ClearTrajectoryStatus(manipulator):
        """
        Clears the current trajectory execution status.
        """
        if not manipulator.simulated:
            manipulator.controller.SendCommand('ClearStatus')

    def MoveUntilTouch(manipulator, direction, distance, max_distance=float('+inf'),
                       max_force=5.0, max_torque=None, ignore_collisions=None, **kw_args):
        """
        Execute a straight move-until-touch action. This action stops when a
        sufficient force is is felt or the manipulator moves the maximum
        distance. The motion is considered successful if the end-effector moves
        at least distance.
        @param direction unit vector for the direction of motion in the world frame
        @param distance minimum distance in meters
        @param max_distance maximum distance in meters
        @param max_force maximum force in Newtons
        @param torque maximum torque in Newton-Meters
        @param execute optionally execute the trajectory
        @param **kw_args planner parameters
        @return felt_force flag indicating whether we felt a force.
        """
        if max_torque is None:
            max_torque = numpy.array([100.0, 100.0, 100.0 ])

        ignore_col_obj_oldstate = []
        if ignore_collisions is None:
            ignore_collisions = []

        for ignore_col_with in ignore_collisions:
            ignore_col_obj_oldstate.append(ignore_col_with.IsEnabled())
            ignore_col_with.Enable(False)

        with manipulator.GetRobot().GetEnv():
            manipulator.GetRobot().GetController().SimulationStep(0)

            # Compute the expected force direction in the hand frame.
            direction = numpy.array(direction)
            hand_pose = manipulator.GetEndEffectorTransform()
            force_direction = numpy.dot(hand_pose[0:3, 0:3].T, -direction)

            with manipulator.GetRobot():
                old_active_manipulator = manipulator.GetRobot().GetActiveManipulator()
                manipulator.SetActive()
                traj = manipulator.PlanToEndEffectorOffset(direction, distance, max_distance=max_distance,
                                                           execute=False, **kw_args)
                traj = manipulator.GetRobot().BlendTrajectory(traj)
                traj = manipulator.GetRobot().RetimeTrajectory(traj, stop_on_ft=True, force_direction=force_direction,
                                                               force_magnitude=max_force, torque=max_torque)

	collided_with_obj = False
        try:
            if not manipulator.simulated:
                manipulator.hand.TareForceTorqueSensor()
                manipulator.GetRobot().ExecuteTrajectory(traj, execute=True, retime=False, blend=False)
                for (ignore_col_with, oldstate) in zip(ignore_collisions, ignore_col_obj_oldstate):
                    ignore_col_with.Enable(oldstate)
            else:
                traj_duration = traj.GetDuration()
                delta_t = 0.01

                traj_config_spec = traj.GetConfigurationSpecification()
                traj_angle_group = traj_config_spec.GetGroupFromName('joint_values')
                path_config_spec = openravepy.ConfigurationSpecification()
#                path_config_spec.AddDeltaTimeGroup()
                path_config_spec.AddGroup(traj_angle_group.name, traj_angle_group.dof, '')
#                path_config_spec.AddDerivativeGroups(1, False);
#                path_config_spec.AddDerivativeGroups(2, False);
#                path_config_spec.AddGroup('owd_blend_radius', 1, 'next')
                path_config_spec.ResetGroupOffsets()

                new_traj = openravepy.RaveCreateTrajectory(manipulator.GetRobot().GetEnv(), '')
                new_traj.Init(path_config_spec)

                for (ignore_col_with, oldstate) in zip(ignore_collisions, ignore_col_obj_oldstate):
                    ignore_col_with.Enable(oldstate)

                with manipulator.GetRobot():
                    manipulator.SetActive()
                    waypoint_ind = 0
                    for t in numpy.arange(0, traj_duration, delta_t):
                        traj_sample = traj.Sample(t)

                        waypoint = traj_config_spec.ExtractJointValues(traj_sample, manipulator.GetRobot(), manipulator.GetArmIndices())
                        manipulator.SetDOFValues(waypoint)
                        if manipulator.GetRobot().GetEnv().CheckCollision(manipulator.GetRobot()):
                            collided_with_obj = True
                            break
                        else:
                            #waypoint = numpy.append(waypoint,t)
                            new_traj.Insert(int(waypoint_ind), waypoint, path_config_spec)
                            waypoint_ind += 1

                    #new_traj = manipulator.GetRobot().BlendTrajectory(new_traj)
                    #new_traj = manipulator.GetRobot().RetimeTrajectory(new_traj)
                manipulator.GetRobot().ExecuteTrajectory(new_traj, execute = True, retime=True, blend=True)

            return collided_with_obj
        # Trajectory is aborted by OWD because we felt a force.
        except exceptions.TrajectoryAborted:
            return True
<|endoftext|>


<commit_before>import unittest
import subprocess
import time
from random import randint
import os

from studio import model
from model_test import get_test_experiment

class HTTPProviderHostedTest(unittest.TestCase):

    def get_db_provider(self, config_name):
        config_file = os.path.join(
            os.path.dirname(
                os.path.realpath(__file__)),
            config_name)
        return model.get_db_provider(model.get_config(config_file))

    def test_add_get_delete_experiment(self):
        with self.get_db_provider('test_config.yaml') as fp, \
                self.get_db_provider('test_config_http_client.yaml') as hp:

            experiment_tuple = get_test_experiment()
            hp.add_experiment(experiment_tuple[0])
            experiment = fp.get_experiment(experiment_tuple[0].key)
            self.assertEquals(experiment.key, experiment_tuple[0].key)
            self.assertEquals(
                experiment.filename,
                experiment_tuple[0].filename)
            self.assertEquals(experiment.args, experiment_tuple[0].args)

            fp.delete_experiment(experiment_tuple[1])

            try:
                thrown = False
                hp.get_experiment(experiment_tuple[1])
            except BaseException:
                thrown = True
            self.assertTrue(thrown)

            experiment_tuple = get_test_experiment()
            fp.add_experiment(experiment_tuple[0])
            experiment = hp.get_experiment(experiment_tuple[0].key)
            self.assertEquals(experiment.key, experiment_tuple[0].key)
            self.assertEquals(
                experiment.filename,
                experiment_tuple[0].filename)
            self.assertEquals(experiment.args, experiment_tuple[0].args)

            hp.delete_experiment(experiment_tuple[1])

            try:
                thrown = False
                fp.get_experiment(experiment_tuple[1])
            except BaseException:
                thrown = True
            self.assertTrue(thrown)

    def test_start_experiment(self):
        with self.get_db_provider('test_config_http_client.yaml') as hp:
            experiment_tuple = get_test_experiment()

            hp.add_experiment(experiment_tuple[0])
            hp.start_experiment(experiment_tuple[0])

            experiment = hp.get_experiment(experiment_tuple[1])

            self.assertTrue(experiment.status == 'running')
            self.assertTrue(experiment.time_added <= time.time())
            self.assertTrue(experiment.time_started <= time.time())

            self.assertEquals(experiment.key, experiment_tuple[0].key)
            self.assertEquals(
                experiment.filename,
                experiment_tuple[0].filename)
            self.assertEquals(experiment.args, experiment_tuple[0].args)

            hp.finish_experiment(experiment_tuple[0])
            hp.delete_experiment(experiment_tuple[1])

if __name__ == '__main__':
    unittest.main()
<commit_msg>removed timestamp comparison in http_provider_hosted_test - fails due to clock mismatch in travis<commit_after>import unittest
import subprocess
import time
from random import randint
import os

from studio import model
from model_test import get_test_experiment

class HTTPProviderHostedTest(unittest.TestCase):

    def get_db_provider(self, config_name):
        config_file = os.path.join(
            os.path.dirname(
                os.path.realpath(__file__)),
            config_name)
        return model.get_db_provider(model.get_config(config_file))

    def test_add_get_delete_experiment(self):
        with self.get_db_provider('test_config.yaml') as fp, \
                self.get_db_provider('test_config_http_client.yaml') as hp:

            experiment_tuple = get_test_experiment()
            hp.add_experiment(experiment_tuple[0])
            experiment = fp.get_experiment(experiment_tuple[0].key)
            self.assertEquals(experiment.key, experiment_tuple[0].key)
            self.assertEquals(
                experiment.filename,
                experiment_tuple[0].filename)
            self.assertEquals(experiment.args, experiment_tuple[0].args)

            fp.delete_experiment(experiment_tuple[1])

            try:
                thrown = False
                hp.get_experiment(experiment_tuple[1])
            except BaseException:
                thrown = True
            self.assertTrue(thrown)

            experiment_tuple = get_test_experiment()
            fp.add_experiment(experiment_tuple[0])
            experiment = hp.get_experiment(experiment_tuple[0].key)
            self.assertEquals(experiment.key, experiment_tuple[0].key)
            self.assertEquals(
                experiment.filename,
                experiment_tuple[0].filename)
            self.assertEquals(experiment.args, experiment_tuple[0].args)

            hp.delete_experiment(experiment_tuple[1])

            try:
                thrown = False
                fp.get_experiment(experiment_tuple[1])
            except BaseException:
                thrown = True
            self.assertTrue(thrown)

    def test_start_experiment(self):
        with self.get_db_provider('test_config_http_client.yaml') as hp:
            experiment_tuple = get_test_experiment()

            hp.add_experiment(experiment_tuple[0])
            hp.start_experiment(experiment_tuple[0])

            experiment = hp.get_experiment(experiment_tuple[1])

            self.assertTrue(experiment.status == 'running')

            self.assertEquals(experiment.key, experiment_tuple[0].key)
            self.assertEquals(
                experiment.filename,
                experiment_tuple[0].filename)
            self.assertEquals(experiment.args, experiment_tuple[0].args)

            hp.finish_experiment(experiment_tuple[0])
            hp.delete_experiment(experiment_tuple[1])

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#  upgrade_tests.py:  test the working copy upgrade process
#
#  Subversion is a tool for revision control.
#  See http://subversion.tigris.org for more information.
#
# ====================================================================
# Copyright (c) 2009 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
######################################################################

#
# These tests exercise the upgrade capabilities of 'svn upgrade' as it
# moves working copies between wc-1 and wc-ng.
#

import os, sys, tarfile, shutil

import svntest

Item = svntest.wc.StateItem
XFail = svntest.testcase.XFail
SkipUnless = svntest.testcase.SkipUnless

def replace_sbox_with_tarfile(sbox, tar_filename):
  try:
    svntest.main.safe_rmtree(sbox.wc_dir)
  except OSError, e:
    pass

  tarpath = os.path.join(os.path.dirname(sys.argv[0]), 'upgrade_tests_data',
                         tar_filename)
  t = tarfile.open(tarpath, 'r:bz2')
  for member in t.getmembers():
    t.extract(member, svntest.main.temp_dir)

  shutil.move(os.path.join(svntest.main.temp_dir, tar_filename.split('.')[0]),
              sbox.wc_dir)

def check_format(sbox, expected_format):
  import sqlite3

  for root, dirs, files in os.walk(sbox.wc_dir):
    db = sqlite3.connect(os.path.join(root, '.svn', 'wc.db'))
    c = db.cursor()
    c.execute('pragma user_version;')
    found_format = c.fetchone()[0]
    db.close()

    if found_format != expected_format:
      raise svntest.Failure("found format '%d'; expected '%d'; in wc '%s'" %
                            (found_format, expected_format, root))

    if '.svn' in dirs:
      dirs.remove('.svn')

def check_dav_cache(dir_path, wc_id, expected_dav_caches):
  import sqlite3

  db = sqlite3.connect(os.path.join(dir_path, '.svn', 'wc.db'))
  c = db.cursor()

  for local_relpath, expected_dav_cache in expected_dav_caches.items():
    c.execute('select dav_cache from base_node ' +
              'where wc_id=? and local_relpath=?',
        (wc_id, local_relpath))
    dav_cache = str(c.fetchone()[0])

    if dav_cache != expected_dav_cache:
      raise svntest.Failure(
              "wrong dav cache for '%s'\n  Found:    '%s'\n  Expected: '%s'" %
                (dir_path, dav_cache, expected_dav_cache))

  db.close()

def basic_upgrade(sbox):
  "basic upgrade behavior"

  replace_sbox_with_tarfile(sbox, 'basic_upgrade.tar.bz2')

  # Attempt to use the working copy, this should give an error
  expected_stderr = (".*Working copy format is too old; please "
                     "run 'svn upgrade'")
  svntest.actions.run_and_verify_svn(None, None, expected_stderr,
                                     'info', sbox.wc_dir)

  # Now upgrade the working copy
  svntest.actions.run_and_verify_svn(None, None, [],
                                     'upgrade', sbox.wc_dir)

  # Actually check the format number of the upgraded working copy
  check_format(sbox, 12)

  # Now check the contents of the working copy
  expected_disk = svntest.main.greek_state.copy()
  expected_status = svntest.actions.get_virginal_state(sbox.wc_dir, 1)
  expected_output = svntest.wc.State(sbox.wc_dir, {})

  # Do the update (which shouldn't pull anything in) and check the statii
  svntest.actions.run_and_verify_update(sbox.wc_dir,
                                        expected_output,
                                        expected_disk,
                                        expected_status,
                                        None, None, None, None, None)

def upgrade_1_5(sbox):
  "test upgrading from a 1.5-era working copy"

  replace_sbox_with_tarfile(sbox, 'upgrade_1_5.tar.bz2')

  # Attempt to use the working copy, this should give an error
  expected_stderr = (".*Working copy format is too old; please "
                     "run 'svn upgrade'")
  svntest.actions.run_and_verify_svn(None, None, expected_stderr,
                                     'info', sbox.wc_dir)

  # Now upgrade the working copy
  svntest.actions.run_and_verify_svn(None, None, [],
                                     'upgrade', sbox.wc_dir)

  # Check the format of the working copy
  check_format(sbox, 12)

  # Now check the contents of the working copy
  expected_disk = svntest.main.greek_state.copy()
  expected_disk.add({
      'A/D/G/E'        : Item(),
      'A/D/G/E/alpha'  : Item("This is the file 'alpha'.\n"),
      'A/D/G/E/beta'   : Item("This is the file 'beta'.\n"),
    })
  expected_status = svntest.actions.get_virginal_state(sbox.wc_dir, 2)
  expected_status.add({
      'A/D/G/E'         : Item(status='  ', wc_rev=2),
      'A/D/G/E/alpha'   : Item(status='  ', wc_rev=2),
      'A/D/G/E/beta'    : Item(status='  ', wc_rev=2),
    })
  expected_output = svntest.wc.State(sbox.wc_dir, {
      'A/D/G/E'        : Item(status='A '),
      'A/D/G/E/alpha'  : Item(status='A '),
      'A/D/G/E/beta'   : Item(status='A '),
    })

  # Do the update (which shouldn't pull anything in) and check the statii
  svntest.actions.run_and_verify_update(sbox.wc_dir,
                                        expected_output,
                                        expected_disk,
                                        expected_status,
                                        None, None, None, None, None)

def logs_left_1_5(sbox):
  "test upgrading from a 1.5-era wc with stale logs"

  replace_sbox_with_tarfile(sbox, 'logs_left_1_5.tar.bz2')

  # Try to upgrade, this should give an error
  expected_stderr = (".*Cannot upgrade with existing logs; please "
                     "run 'svn cleanup' with Subversion 1.6")
  svntest.actions.run_and_verify_svn(None, None, expected_stderr,
                                     'upgrade', sbox.wc_dir)

def upgrade_wcprops(sbox):
  "test upgrading a working copy with wcprops"

  replace_sbox_with_tarfile(sbox, 'upgrade_wcprops.tar.bz2')
  svntest.actions.run_and_verify_svn(None, None, [],
                                     'upgrade', sbox.wc_dir)

  # Make sure that .svn/all-wcprops has disappeared
  if os.path.exists(os.path.join(sbox.wc_dir, '.svn', 'all-wcprops')):
    raise svntest.Failure("all-wcprops file still exists")

  # Just for kicks, let's see if the wcprops are what we'd expect them
  # to be.  (This could be smarter.)
  expected_dav_caches = {
   '' :
    '(svn:wc:ra_dav:version-url 41 /svn-test-work/local_tmp/repos/!svn/ver/1)',
   'iota' :
    '(svn:wc:ra_dav:version-url 46 /svn-test-work/local_tmp/repos/!svn/ver/1/iota)',
  }
  check_dav_cache(sbox.wc_dir, 1, expected_dav_caches)

########################################################################
# Run the tests

def has_sqlite():
  try:
    import sqlite3
    return True
  except ImportError:
    return False

# list all tests here, starting with None:
test_list = [ None,
              SkipUnless(basic_upgrade, has_sqlite),
              SkipUnless(upgrade_1_5, has_sqlite),
              logs_left_1_5,
              SkipUnless(upgrade_wcprops, has_sqlite),
             ]

if __name__ == '__main__':
  svntest.main.run_tests(test_list)
  # NOTREACHED
<commit_msg>Fix upgrade tests 1 and 2.  It turns out that run_and_verify_update() eventually invokes run_and_verify_status(), which is what we were after in the first place.  That function, eventually runs 'svn st -v -u -q', which requires server access.  Since the dummy working copies used in these tests don't have corresponding repositories, those calls fail.<commit_after>#!/usr/bin/env python
#
#  upgrade_tests.py:  test the working copy upgrade process
#
#  Subversion is a tool for revision control.
#  See http://subversion.tigris.org for more information.
#
# ====================================================================
# Copyright (c) 2009 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
######################################################################

#
# These tests exercise the upgrade capabilities of 'svn upgrade' as it
# moves working copies between wc-1 and wc-ng.
#

import os, sys, tarfile, shutil

import svntest

Item = svntest.wc.StateItem
XFail = svntest.testcase.XFail
SkipUnless = svntest.testcase.SkipUnless

def replace_sbox_with_tarfile(sbox, tar_filename):
  try:
    svntest.main.safe_rmtree(sbox.wc_dir)
  except OSError, e:
    pass

  tarpath = os.path.join(os.path.dirname(sys.argv[0]), 'upgrade_tests_data',
                         tar_filename)
  t = tarfile.open(tarpath, 'r:bz2')
  for member in t.getmembers():
    t.extract(member, svntest.main.temp_dir)

  shutil.move(os.path.join(svntest.main.temp_dir, tar_filename.split('.')[0]),
              sbox.wc_dir)

def check_format(sbox, expected_format):
  import sqlite3

  for root, dirs, files in os.walk(sbox.wc_dir):
    db = sqlite3.connect(os.path.join(root, '.svn', 'wc.db'))
    c = db.cursor()
    c.execute('pragma user_version;')
    found_format = c.fetchone()[0]
    db.close()

    if found_format != expected_format:
      raise svntest.Failure("found format '%d'; expected '%d'; in wc '%s'" %
                            (found_format, expected_format, root))

    if '.svn' in dirs:
      dirs.remove('.svn')

def check_dav_cache(dir_path, wc_id, expected_dav_caches):
  import sqlite3

  db = sqlite3.connect(os.path.join(dir_path, '.svn', 'wc.db'))
  c = db.cursor()

  for local_relpath, expected_dav_cache in expected_dav_caches.items():
    c.execute('select dav_cache from base_node ' +
              'where wc_id=? and local_relpath=?',
        (wc_id, local_relpath))
    dav_cache = str(c.fetchone()[0])

    if dav_cache != expected_dav_cache:
      raise svntest.Failure(
              "wrong dav cache for '%s'\n  Found:    '%s'\n  Expected: '%s'" %
                (dir_path, dav_cache, expected_dav_cache))

  db.close()

def run_and_verify_status_no_server(wc_dir, expected_status):
  "same as svntest.actions.run_and_verify_status(), but without '-u'"

  exit_code, output, errput = svntest.main.run_svn(None, 'st', '-q', '-v',
                                                   wc_dir)
  actual = svntest.tree.build_tree_from_status(output)
  try:
    svntest.tree.compare_trees("status", actual, expected_status.old_tree())
  except svntest.tree.SVNTreeError:
    svntest.verify.display_trees(None, 'STATUS OUTPUT TREE',
                                 output_tree, actual)
    print("ACTUAL STATUS TREE:")
    svvtest.tree.dump_tree_script(actual, wc_dir_name + os.sep)
    raise

def basic_upgrade(sbox):
  "basic upgrade behavior"

  replace_sbox_with_tarfile(sbox, 'basic_upgrade.tar.bz2')

  # Attempt to use the working copy, this should give an error
  expected_stderr = (".*Working copy format is too old; please "
                     "run 'svn upgrade'")
  svntest.actions.run_and_verify_svn(None, None, expected_stderr,
                                     'info', sbox.wc_dir)

  # Now upgrade the working copy
  svntest.actions.run_and_verify_svn(None, None, [],
                                     'upgrade', sbox.wc_dir)

  # Actually check the format number of the upgraded working copy
  check_format(sbox, 12)

  # Now check the contents of the working copy
  expected_status = svntest.actions.get_virginal_state(sbox.wc_dir, 1)
  run_and_verify_status_no_server(sbox.wc_dir, expected_status)

def upgrade_1_5(sbox):
  "test upgrading from a 1.5-era working copy"

  replace_sbox_with_tarfile(sbox, 'upgrade_1_5.tar.bz2')

  # Attempt to use the working copy, this should give an error
  expected_stderr = (".*Working copy format is too old; please "
                     "run 'svn upgrade'")
  svntest.actions.run_and_verify_svn(None, None, expected_stderr,
                                     'info', sbox.wc_dir)

  # Now upgrade the working copy
  svntest.actions.run_and_verify_svn(None, None, [],
                                     'upgrade', sbox.wc_dir)

  # Check the format of the working copy
  check_format(sbox, 12)

  # Now check the contents of the working copy
  expected_status = svntest.actions.get_virginal_state(sbox.wc_dir, 1)
  run_and_verify_status_no_server(sbox.wc_dir, expected_status)

def logs_left_1_5(sbox):
  "test upgrading from a 1.5-era wc with stale logs"

  replace_sbox_with_tarfile(sbox, 'logs_left_1_5.tar.bz2')

  # Try to upgrade, this should give an error
  expected_stderr = (".*Cannot upgrade with existing logs; please "
                     "run 'svn cleanup' with Subversion 1.6")
  svntest.actions.run_and_verify_svn(None, None, expected_stderr,
                                     'upgrade', sbox.wc_dir)

def upgrade_wcprops(sbox):
  "test upgrading a working copy with wcprops"

  replace_sbox_with_tarfile(sbox, 'upgrade_wcprops.tar.bz2')
  svntest.actions.run_and_verify_svn(None, None, [],
                                     'upgrade', sbox.wc_dir)

  # Make sure that .svn/all-wcprops has disappeared
  if os.path.exists(os.path.join(sbox.wc_dir, '.svn', 'all-wcprops')):
    raise svntest.Failure("all-wcprops file still exists")

  # Just for kicks, let's see if the wcprops are what we'd expect them
  # to be.  (This could be smarter.)
  expected_dav_caches = {
   '' :
    '(svn:wc:ra_dav:version-url 41 /svn-test-work/local_tmp/repos/!svn/ver/1)',
   'iota' :
    '(svn:wc:ra_dav:version-url 46 /svn-test-work/local_tmp/repos/!svn/ver/1/iota)',
  }
  check_dav_cache(sbox.wc_dir, 1, expected_dav_caches)

########################################################################
# Run the tests

def has_sqlite():
  try:
    import sqlite3
    return True
  except ImportError:
    return False

# list all tests here, starting with None:
test_list = [ None,
              SkipUnless(basic_upgrade, has_sqlite),
              SkipUnless(upgrade_1_5, has_sqlite),
              logs_left_1_5,
              SkipUnless(upgrade_wcprops, has_sqlite),
             ]

if __name__ == '__main__':
  svntest.main.run_tests(test_list)
  # NOTREACHED
<|endoftext|>


<commit_before>"""
Generic LightCurve Tests
"""
from __future__ import absolute_import

#
# @TODO:
#   ndarray + indices?
#   time deltas instead of datetimes?

#pylint: disable=C0103,R0904,W0201,W0232,E1101,E1103
import numpy as np
import pytest
import datetime
import sunpy
from sunpy.data.test import (EVE_AVERAGES_CSV)

def test_input_empty():
    """Tests empty input"""
    with pytest.raises(NotImplementedError):
        sunpy.lightcurve.LightCurve()

def test_input_dict_datetimes():
    """Tests LightCurve creation from a dictionary and list of datetimes"""
    base = datetime.datetime.today()
    dates = [base - datetime.timedelta(minutes=x) for x in range(0, 24 * 60)]
    sunpy.lightcurve.LightCurve({"param": range(24 * 60)}, index=dates)

def test_input_ndarray_datetimes():
    """Tests LightCurve creation from a dictionary and list of datetimes"""
    base = datetime.datetime.today()
    dates = [base - datetime.timedelta(minutes=x) for x in range(0, 24 * 60)]
    sunpy.lightcurve.LightCurve(np.arange(24 * 60), index=dates)

def test_input_file():
    """Tests filepath input"""
    with pytest.raises(NotImplementedError):
        sunpy.lightcurve.LightCurve(EVE_AVERAGES_CSV)<commit_msg>list + dates input unit test<commit_after>"""
Generic LightCurve Tests
"""
from __future__ import absolute_import

#
# @TODO:
#   ndarray + indices?
#   time deltas instead of datetimes?

#pylint: disable=C0103,R0904,W0201,W0232,E1101,E1103
import numpy as np
import pytest
import datetime
import sunpy
import pandas
from sunpy.data.test import (EVE_AVERAGES_CSV)

def test_input_empty():
    """Tests empty input"""
    with pytest.raises(NotImplementedError):
        sunpy.lightcurve.LightCurve()

def test_input_dict_datetimes():
    """Tests LightCurve creation from a dictionary and list of datetimes"""
    base = datetime.datetime.today()
    dates = [base - datetime.timedelta(minutes=x) for x in range(0, 24 * 60)]
    sunpy.lightcurve.LightCurve({"param": range(24 * 60)}, index=dates)

def test_input_ndarray_datetimes():
    """Tests LightCurve creation from a dictionary and list of datetimes"""
    base = datetime.datetime.today()
    dates = [base - datetime.timedelta(minutes=x) for x in range(0, 24 * 60)]
    sunpy.lightcurve.LightCurve(np.arange(24 * 60), index=dates)

def test_input_list_datetimes():
    """Tests LightCurve creation from a dictionary and list of datetimes"""
    base = datetime.datetime.today()
    dates = [base - datetime.timedelta(minutes=x) for x in range(0, 24 * 60)]
    sunpy.lightcurve.LightCurve(range(24 * 60), index=dates)

def test_input_file():
    """Tests filepath input"""
    with pytest.raises(NotImplementedError):
        sunpy.lightcurve.LightCurve(EVE_AVERAGES_CSV)

<|endoftext|>


<commit_before>from os import listdir

# Last dumped match
last_match = lambda path: max([int(m.split('.')[0]) for m in listdir(path)])

# Get one here: https://developer.riotgames.com/
API_KEY = 'SUA_API_AQUI'

# Direcotry where matches will be saved
DUMP_DIR = '../dump/'

# Direcotry where parsed matches will be saved
DATA_DIR = '../data/'

# Dictionary of first matches by season
FIRST_MATCH_ID = {
    'SEASON2015': 456157041  # first match of season 2015
}

# Filter used in the parser
MATCH_FILTER = {
    'matchMode': 'CLASSIC',
    'season': 'SEASON2015',
    'region': 'BR',
    'queueType': 'RANKED_SOLO_5x5',
}

# First match to dump
try:
    STARTING_MATCH_ID = last_match(DUMP_DIR)
except ValueError:
    STARTING_MATCH_ID = FIRST_MATCH_ID['SEASON2015']
<commit_msg>Cleaning config.<commit_after># Get one here: https://developer.riotgames.com/
API_KEY = 'SUA_API_AQUI'

# Direcotry where matches will be saved
DUMP_DIR = '../dump/'

# Direcotry where parsed matches will be saved
DATA_DIR = '../data/'

# Dictionary of first matches by season
FIRST_MATCH_ID = {
    'SEASON2015': 456157041  # first match of season 2015
}

# Filter used in the parser
MATCH_FILTER = {
    'matchMode': 'CLASSIC',
    'season': 'SEASON2015',
    'region': 'BR',
    'queueType': 'RANKED_SOLO_5x5',
}

# First match to dump
STARTING_MATCH_ID = FIRST_MATCH_ID['SEASON2015']
<|endoftext|>


<commit_before>import click
from .manager.database import update
from .webserver.web import get_app

@click.group()
def main():
    pass

@main.command()
def update():
    """Update PyUniProt data"""
    pyuniprot.update()

@main.command()
def web():
    get_app().run()

<commit_msg>Fix typo<commit_after>import click
from .manager.database import update
from .webserver.web import get_app

@click.group()
def main():
    pass

@main.command()
def update():
    """Update PyUniProt data"""
    update()

@main.command()
def web():
    get_app().run()

<|endoftext|>


<commit_before># coding=utf-8
# Distributed under the MIT software license, see the accompanying
# file LICENSE or http://www.opensource.org/licenses/mit-license.php.
import simplejson as json
import grpc
from google.protobuf.json_format import MessageToJson
from qrl.core import config
from qrl.core.AddressState import AddressState
from qrl.crypto.xmss import XMSS
from qrl.core.Transaction import TransferTransaction, Transaction
from pyqrllib.pyqrllib import hstr2bin, bin2hstr

from qrl.generated import qrl_pb2_grpc, qrl_pb2, qrlmining_pb2, qrlmining_pb2_grpc
from flask import Flask, Response, request
from jsonrpc.backend.flask import api

app = Flask(__name__)

def read_slaves(slaves_filename):
    with open(slaves_filename, 'r') as f:
        slave_data = json.load(f)
        slave_data[0] = bytes(hstr2bin(slave_data[0]))
        return slave_data

def get_addr_state(addr: bytes) -> AddressState:
    stub = get_public_stub()
    response = stub.GetAddressState(request=qrl_pb2.GetAddressStateReq(address=addr))
    return AddressState(response.state)

def set_unused_ots_key(xmss, addr_state, start=0):
    for i in range(start, 2 ** xmss.height):
        if not addr_state.ots_key_reuse(i):
            xmss.set_ots_index(i)
            return True
    return False

def valid_payment_permission(public_stub, master_address_state, payment_xmss, json_slave_txn):
    access_type = master_address_state.get_slave_permission(payment_xmss.pk)

    if access_type == -1:
        tx = Transaction.from_json(json_slave_txn)
        public_stub.PushTransaction(request=qrl_pb2.PushTransactionReq(transaction_signed=tx.pbdata))
        return None

    if access_type == 0:
        return True

    return False

def get_unused_payment_xmss(public_stub):
    global payment_slaves
    global payment_xmss

    master_address = payment_slaves[0]
    master_address_state = get_addr_state(master_address)

    if payment_xmss:
        addr_state = get_addr_state(payment_xmss.address)
        if set_unused_ots_key(payment_xmss, addr_state, payment_xmss.ots_index):
            if valid_payment_permission(public_stub, master_address_state, payment_xmss, payment_slaves[2]):
                return payment_xmss
        else:
            payment_xmss = None

    if not payment_xmss:
        unused_ots_found = False
        for slave_seed in payment_slaves[1]:
            xmss = XMSS.from_extended_seed(slave_seed)
            addr_state = get_addr_state(xmss.address)
            if set_unused_ots_key(xmss, addr_state):  # Unused ots_key_found
                payment_xmss = xmss
                unused_ots_found = True
                break

        if not unused_ots_found:  # Unused ots_key_found
            return None

    if not valid_payment_permission(public_stub, master_address_state, payment_xmss, payment_slaves[2]):
        return None

    return payment_xmss

@app.route('/api/<api_method_name>')
def api_proxy(api_method_name):
    """
    Proxy JSON RPC requests to the gRPC server as well as converts back gRPC response
    to JSON.
    TODO :
    1.  Remove hardcoded Server IP and Port
    :param api_method_name:
    :return:
    """
    stub = qrl_pb2_grpc.PublicAPIStub(grpc.insecure_channel('127.0.0.1:9009'))
    public_api = qrl_pb2.DESCRIPTOR.services_by_name['PublicAPI']
    api_method = public_api.FindMethodByName(api_method_name)
    api_request = getattr(qrl_pb2, api_method.input_type.name)()

    for arg in request.args:
        if arg not in api_method.input_type.fields_by_name:
            raise Exception('Invalid args %s', arg)
        data_type = type(getattr(api_request, arg))
        if data_type == bool and request.args[arg].lower() == 'false':
            continue
        value = data_type(request.args.get(arg, type=data_type))
        setattr(api_request, arg, value)

    resp = getattr(stub, api_method_name)(api_request, timeout=10)
    return Response(response=MessageToJson(resp), status=200, mimetype='application/json')

def get_mining_stub():
    global mining_stub
    return mining_stub

def get_public_stub():
    global public_stub
    return public_stub

@api.dispatcher.add_method
def getlastblockheader(height=0):
    stub = get_mining_stub()
    request = qrlmining_pb2.GetLastBlockHeaderReq(height=height)
    grpc_response = stub.GetLastBlockHeader(request=request, timeout=10)

    block_header = {
        'difficulty': grpc_response.difficulty,
        'height': grpc_response.height,
        'timestamp': grpc_response.timestamp,
        'reward': grpc_response.reward,
        'hash': grpc_response.hash,
        'depth': grpc_response.depth
    }

    resp = {
        "block_header": block_header,
        "status": "OK"
    }
    return resp

@api.dispatcher.add_method
def getblockheaderbyheight(height):
    return getlastblockheader(height)

@api.dispatcher.add_method
def getblocktemplate(wallet_address):
    stub = get_mining_stub()
    request = qrlmining_pb2.GetBlockToMineReq(wallet_address=wallet_address.encode())
    grpc_response = stub.GetBlockToMine(request=request, timeout=10)
    resp = {
        'blocktemplate_blob': grpc_response.blocktemplate_blob,
        'difficulty': grpc_response.difficulty,
        'height': grpc_response.height,
        'reserved_offset': grpc_response.reserved_offset,
        'status': 'OK'
    }

    return resp

@api.dispatcher.add_method
def submitblock(blob):
    stub = get_mining_stub()
    request = qrlmining_pb2.SubmitMinedBlockReq(blob=bytes(hstr2bin(blob)))
    response = stub.SubmitMinedBlock(request=request, timeout=10)
    if response.error:
        raise Exception  # Mining pool expected exception when block submission fails
    return MessageToJson(response)

@api.dispatcher.add_method
def getblockminingcompatible(height):
    stub = get_mining_stub()
    request = qrlmining_pb2.GetBlockMiningCompatibleReq(height=height)
    response = stub.GetBlockMiningCompatible(request=request, timeout=10)
    return MessageToJson(response)

@api.dispatcher.add_method
def transfer(destinations, fee, mixin, unlock_time):
    if len(destinations) > config.dev.transaction_multi_output_limit:
        return None

    addrs_to = []
    amounts = []

    for tx in destinations:
        addrs_to.append(bytes(hstr2bin(tx['address'][1:])))  # Skipping 'Q'
        amounts.append(tx['amount'])

    stub = get_public_stub()

    xmss = get_unused_payment_xmss(stub)
    if not xmss:
        return None

    tx = TransferTransaction.create(addrs_to=addrs_to,
                                    amounts=amounts,
                                    fee=fee,
                                    xmss_pk=xmss.pk,
                                    master_addr=payment_slaves[0])

    tx.sign(xmss)

    response = stub.PushTransaction(request=qrl_pb2.PushTransactionReq(transaction_signed=tx.pbdata))

    if response.error_code != 3:
        return None

    response = {'tx_hash': bin2hstr(tx.txhash)}

    return response

app.add_url_rule('/json_rpc', 'api', api.as_view(), methods=['POST'])

def main():
    global payment_slaves, payment_xmss
    global mining_stub, public_stub
    mining_stub = qrlmining_pb2_grpc.MiningAPIStub(grpc.insecure_channel('127.0.0.1:9007'))
    public_stub = qrl_pb2_grpc.PublicAPIStub(grpc.insecure_channel('127.0.0.1:9009'))
    payment_xmss = None
    payment_slaves = read_slaves(config.user.mining_pool_payment_wallet_path)
    app.run(host='127.0.0.1', port=18081)

if __name__ == '__main__':
    main()
<commit_msg>Fix: Added missing argument in grpcProxy<commit_after># coding=utf-8
# Distributed under the MIT software license, see the accompanying
# file LICENSE or http://www.opensource.org/licenses/mit-license.php.
import simplejson as json
import grpc
from google.protobuf.json_format import MessageToJson
from qrl.core import config
from qrl.core.AddressState import AddressState
from qrl.crypto.xmss import XMSS
from qrl.core.Transaction import TransferTransaction, Transaction
from pyqrllib.pyqrllib import hstr2bin, bin2hstr

from qrl.generated import qrl_pb2_grpc, qrl_pb2, qrlmining_pb2, qrlmining_pb2_grpc
from flask import Flask, Response, request
from jsonrpc.backend.flask import api

app = Flask(__name__)

def read_slaves(slaves_filename):
    with open(slaves_filename, 'r') as f:
        slave_data = json.load(f)
        slave_data[0] = bytes(hstr2bin(slave_data[0]))
        return slave_data

def get_addr_state(addr: bytes) -> AddressState:
    stub = get_public_stub()
    response = stub.GetAddressState(request=qrl_pb2.GetAddressStateReq(address=addr))
    return AddressState(response.state)

def set_unused_ots_key(xmss, addr_state, start=0):
    for i in range(start, 2 ** xmss.height):
        if not addr_state.ots_key_reuse(i):
            xmss.set_ots_index(i)
            return True
    return False

def valid_payment_permission(public_stub, master_address_state, payment_xmss, json_slave_txn):
    access_type = master_address_state.get_slave_permission(payment_xmss.pk)

    if access_type == -1:
        tx = Transaction.from_json(json_slave_txn)
        public_stub.PushTransaction(request=qrl_pb2.PushTransactionReq(transaction_signed=tx.pbdata))
        return None

    if access_type == 0:
        return True

    return False

def get_unused_payment_xmss(public_stub):
    global payment_slaves
    global payment_xmss

    master_address = payment_slaves[0]
    master_address_state = get_addr_state(master_address)

    if payment_xmss:
        addr_state = get_addr_state(payment_xmss.address)
        if set_unused_ots_key(payment_xmss, addr_state, payment_xmss.ots_index):
            if valid_payment_permission(public_stub, master_address_state, payment_xmss, payment_slaves[2]):
                return payment_xmss
        else:
            payment_xmss = None

    if not payment_xmss:
        unused_ots_found = False
        for slave_seed in payment_slaves[1]:
            xmss = XMSS.from_extended_seed(slave_seed)
            addr_state = get_addr_state(xmss.address)
            if set_unused_ots_key(xmss, addr_state):  # Unused ots_key_found
                payment_xmss = xmss
                unused_ots_found = True
                break

        if not unused_ots_found:  # Unused ots_key_found
            return None

    if not valid_payment_permission(public_stub, master_address_state, payment_xmss, payment_slaves[2]):
        return None

    return payment_xmss

@app.route('/api/<api_method_name>')
def api_proxy(api_method_name):
    """
    Proxy JSON RPC requests to the gRPC server as well as converts back gRPC response
    to JSON.
    TODO :
    1.  Remove hardcoded Server IP and Port
    :param api_method_name:
    :return:
    """
    stub = qrl_pb2_grpc.PublicAPIStub(grpc.insecure_channel('127.0.0.1:9009'))
    public_api = qrl_pb2.DESCRIPTOR.services_by_name['PublicAPI']
    api_method = public_api.FindMethodByName(api_method_name)
    api_request = getattr(qrl_pb2, api_method.input_type.name)()

    for arg in request.args:
        if arg not in api_method.input_type.fields_by_name:
            raise Exception('Invalid args %s', arg)
        data_type = type(getattr(api_request, arg))
        if data_type == bool and request.args[arg].lower() == 'false':
            continue
        value = data_type(request.args.get(arg, type=data_type))
        setattr(api_request, arg, value)

    resp = getattr(stub, api_method_name)(api_request, timeout=10)
    return Response(response=MessageToJson(resp), status=200, mimetype='application/json')

def get_mining_stub():
    global mining_stub
    return mining_stub

def get_public_stub():
    global public_stub
    return public_stub

@api.dispatcher.add_method
def getlastblockheader(height=0):
    stub = get_mining_stub()
    request = qrlmining_pb2.GetLastBlockHeaderReq(height=height)
    grpc_response = stub.GetLastBlockHeader(request=request, timeout=10)

    block_header = {
        'difficulty': grpc_response.difficulty,
        'height': grpc_response.height,
        'timestamp': grpc_response.timestamp,
        'reward': grpc_response.reward,
        'hash': grpc_response.hash,
        'depth': grpc_response.depth
    }

    resp = {
        "block_header": block_header,
        "status": "OK"
    }
    return resp

@api.dispatcher.add_method
def getblockheaderbyheight(height):
    return getlastblockheader(height)

@api.dispatcher.add_method
def getblocktemplate(reserve_size, wallet_address):
    stub = get_mining_stub()
    request = qrlmining_pb2.GetBlockToMineReq(wallet_address=wallet_address.encode())
    grpc_response = stub.GetBlockToMine(request=request, timeout=10)
    resp = {
        'blocktemplate_blob': grpc_response.blocktemplate_blob,
        'difficulty': grpc_response.difficulty,
        'height': grpc_response.height,
        'reserved_offset': grpc_response.reserved_offset,
        'status': 'OK'
    }

    return resp

@api.dispatcher.add_method
def submitblock(blob):
    stub = get_mining_stub()
    request = qrlmining_pb2.SubmitMinedBlockReq(blob=bytes(hstr2bin(blob)))
    response = stub.SubmitMinedBlock(request=request, timeout=10)
    if response.error:
        raise Exception  # Mining pool expected exception when block submission fails
    return MessageToJson(response)

@api.dispatcher.add_method
def getblockminingcompatible(height):
    stub = get_mining_stub()
    request = qrlmining_pb2.GetBlockMiningCompatibleReq(height=height)
    response = stub.GetBlockMiningCompatible(request=request, timeout=10)
    return MessageToJson(response)

@api.dispatcher.add_method
def transfer(destinations, fee, mixin, unlock_time):
    if len(destinations) > config.dev.transaction_multi_output_limit:
        return None

    addrs_to = []
    amounts = []

    for tx in destinations:
        addrs_to.append(bytes(hstr2bin(tx['address'][1:])))  # Skipping 'Q'
        amounts.append(tx['amount'])

    stub = get_public_stub()

    xmss = get_unused_payment_xmss(stub)
    if not xmss:
        return None

    tx = TransferTransaction.create(addrs_to=addrs_to,
                                    amounts=amounts,
                                    fee=fee,
                                    xmss_pk=xmss.pk,
                                    master_addr=payment_slaves[0])

    tx.sign(xmss)

    response = stub.PushTransaction(request=qrl_pb2.PushTransactionReq(transaction_signed=tx.pbdata))

    if response.error_code != 3:
        return None

    response = {'tx_hash': bin2hstr(tx.txhash)}

    return response

app.add_url_rule('/json_rpc', 'api', api.as_view(), methods=['POST'])

def main():
    global payment_slaves, payment_xmss
    global mining_stub, public_stub
    mining_stub = qrlmining_pb2_grpc.MiningAPIStub(grpc.insecure_channel('127.0.0.1:9007'))
    public_stub = qrl_pb2_grpc.PublicAPIStub(grpc.insecure_channel('127.0.0.1:9009'))
    payment_xmss = None
    payment_slaves = read_slaves(config.user.mining_pool_payment_wallet_path)
    app.run(host='127.0.0.1', port=18081)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! -*- coding: utf-8 -*-
# package modules
from JapaneseTokenizer.object_models import WrapperBase
from JapaneseTokenizer.common import text_preprocess, juman_utils
from JapaneseTokenizer import init_logger
from JapaneseTokenizer.datamodels import FilteredObject, TokenizedSenetence
from JapaneseTokenizer.common.sever_handler import JumanppHnadler  # todo
# pyknp modules
from pyknp.juman.mlist import MList
# else
from typing import List, Union, TypeVar, Tuple, Callable
from six import text_type
import logging
import sys
import socket
import six
import os
import re
__author__ = 'kensuke-mi'

logger = init_logger.init_logger(logging.getLogger(init_logger.LOGGER_NAME))
python_version = sys.version_info
ContentsTypes = TypeVar('T')

try:
    import pyknp
except ImportError:
    logger.warning(msg='pyknp is not ready to use. Install first if you would like to use pyknp wrapper.')

class MonkeyPatchSocket(object):
    """* Class for overwriting pyknp.Socket because it is only for python2.x"""
    def __init__(self, hostname, port, option=None):
        try:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.connect((hostname, port))
        except:
            raise
        if option is not None:
            self.sock.send(option)
        data = b""
        while b"OK" not in data:
            #while isinstance(data, bytes) and b"OK" not in data:
            data = self.sock.recv(1024)

    def __del__(self):
        if self.sock:
            self.sock.close()

    def query(self, sentence, pattern):
        assert(isinstance(sentence, six.text_type))
        self.sock.sendall(b"%s\n" % sentence.encode('utf-8').strip())
        data = self.sock.recv(1024)
        assert isinstance(data, bytes)
        recv = data
        while not re.search(pattern, recv):
            data = self.sock.recv(1024)
            recv = b"%s%s" % (recv, data)
        return recv.strip().decode('utf-8')

class JumanWrapper(WrapperBase):
    def __init__(self,
                 command='juman',
                 server=None,
                 port=32000,
                 timeout=30,
                 rcfile=None,
                 option='-e2 -B',
                 pattern='EOS',
                 is_use_pyknp=False,
                 **args):
        """* Class to call Juman tokenizer
        """
        # type: (text_type,Union[str,None],int,int,text_type,Union[bytes,str],Union[bytes,str],bool)->None

        self.timeout = timeout
        self.pattern = pattern
        self.option = option
        self.command = command
        if not rcfile is None and not os.path.exists(rcfile):
            raise FileExistsError('rcfile does not exist at {}'.format(rcfile))
        if not server is None:
            ### It converts from str into bytes only for sever mode ###
            self.option = self.option.encode('utf-8')
            self.pattern = self.pattern.encode('utf-8')
        else:
            pass

        # check os #
        if os.name == 'nt':
            if not is_use_pyknp:
                logger.warning(msg='It forces is_use_pyknp = True on Windows.')
            else:
                pass
            self.is_use_pyknp = True
        else:
            pass

        if is_use_pyknp:
            self.juman = pyknp.Juman(command=command, server=server, port=port,
                                     timeout=self.timeout, rcfile=rcfile, option=option,
                                     pattern=pattern, **args)
            ### It overwrites juman_lines() method ###
            self.juman.juman_lines = self.__monkey_patch_juman_lines
        else:
            self.juman = JumanppHnadler(jumanpp_command=command,
                                        option=self.option,
                                        pattern=self.pattern,
                                        timeout_second=self.timeout)

    def __del__(self):
        if hasattr(self, "juman"):
            if isinstance(self.juman, JumanppHnadler):
                self.juman.stop_process()

    def __monkey_patch_juman_lines(self, input_str:str):
        """* What you can do
        - It overwrites juman_line() method because this method causes TypeError in python3
        """
        assert isinstance(self.juman, pyknp.Juman)
        if not self.juman.socket and not self.juman.subprocess:
            if self.juman.server is not None:
                self.juman.socket = MonkeyPatchSocket(self.juman.server, self.juman.port, b"RUN -e2\n")
            else:
                command = "%s %s" % (self.juman.command, self.juman.option)
                if self.juman.rcfile:
                    command += " -r %s" % self.juman.rcfile
                self.juman.subprocess = pyknp.Subprocess(command)
        if self.juman.socket:
            return self.juman.socket.query(input_str, pattern=self.juman.pattern)
        return self.juman.subprocess.query(input_str, pattern=self.juman.pattern)

    def call_juman_interface(self, input_str):
        """* What you can do
        - You call Juman tokenizer interface.

        * Output
        - pyknp.MList
        """
        # type: (str)->MList
        if isinstance(self.juman, pyknp.Juman):
            return self.juman.analysis(input_str)
        elif isinstance(self.juman, JumanppHnadler):
            result_analysis = self.juman.query(input_str)
            return MList(spec=result_analysis)

    def tokenize(self, sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (str, bool, bool, bool, bool, Callable[[str], str]) -> Union[TokenizedSenetence, List[str]]
        """
        :param sentence:
        :param ins_mecab:
        :param list_stopword:
        :param list_pos_candidate:
        :return:  list [tuple (unicode, unicode)]
        """
        assert isinstance(normalize, bool)
        assert isinstance(sentence, str)
        if normalize:
            normalized_sentence = func_normalizer(sentence)
        else:
            normalized_sentence = sentence

        result = self.call_juman_interface(normalized_sentence)
        token_objects = [
            juman_utils.extract_morphological_information(
                mrph_object=morph_object,
                is_surface=is_surface,
                is_feature=is_feature
            )
            for morph_object in result]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)

            return tokenized_objects

    def filter(self, parsed_sentence:TokenizedSenetence, pos_condition:List[Tuple[str, ...]]=None, stopwords:List[str]=None)->FilteredObject:
        assert isinstance(parsed_sentence, TokenizedSenetence)
        assert isinstance(pos_condition, (type(None), list))
        assert isinstance(stopwords, (type(None), list))

        return parsed_sentence.filter(pos_condition, stopwords)
<commit_msg>fixed a bug in juman server mode for python3<commit_after>#! -*- coding: utf-8 -*-
# package modules
from JapaneseTokenizer.object_models import WrapperBase
from JapaneseTokenizer.common import text_preprocess, juman_utils
from JapaneseTokenizer import init_logger
from JapaneseTokenizer.datamodels import FilteredObject, TokenizedSenetence
from JapaneseTokenizer.common.sever_handler import JumanppHnadler  # todo
# pyknp modules
from pyknp.juman.mlist import MList
# else
from typing import List, Union, TypeVar, Tuple, Callable
from six import text_type
import logging
import sys
import socket
import six
import os
import re
__author__ = 'kensuke-mi'

logger = init_logger.init_logger(logging.getLogger(init_logger.LOGGER_NAME))
python_version = sys.version_info
ContentsTypes = TypeVar('T')

try:
    import pyknp
except ImportError:
    logger.warning(msg='pyknp is not ready to use. Install first if you would like to use pyknp wrapper.')

class MonkeyPatchSocket(object):
    """* Class for overwriting pyknp.Socket because it is only for python2.x"""
    def __init__(self, hostname, port, option=None):
        try:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.connect((hostname, port))
        except:
            raise
        if option is not None:
            self.sock.send(option)
        data = b""
        while b"OK" not in data:
            #while isinstance(data, bytes) and b"OK" not in data:
            data = self.sock.recv(1024)

    def __del__(self):
        if self.sock:
            self.sock.close()

    def query(self, sentence, pattern):
        """"""
        # type: (str,str)->str
        assert(isinstance(sentence, six.text_type))
        sentence_bytes = sentence.encode('utf-8').strip()
        pattern_bytes = pattern.encode('utf-8')

        self.sock.sendall(sentence_bytes + b"\n")
        data = self.sock.recv(1024)
        assert isinstance(data, bytes)
        recv = data
        while not re.search(pattern_bytes, recv):
            data = self.sock.recv(1024)
            recv = recv + data
        return recv.strip().decode('utf-8')

class JumanWrapper(WrapperBase):
    def __init__(self,
                 command='juman',
                 server=None,
                 port=32000,
                 timeout=30,
                 rcfile=None,
                 option='-e2 -B',
                 pattern='EOS',
                 is_use_pyknp=False,
                 **args):
        """* Class to call Juman tokenizer
        """
        # type: (text_type,Union[str,None],int,int,text_type,Union[bytes,str],Union[bytes,str],bool)->None

        self.timeout = timeout
        self.pattern = pattern
        self.option = option
        self.command = command
        if not rcfile is None and not os.path.exists(rcfile):
            raise FileExistsError('rcfile does not exist at {}'.format(rcfile))
        if not server is None:
            ### It converts from str into bytes only for sever mode ###
            self.option = self.option.encode('utf-8')
            self.pattern = self.pattern.encode('utf-8')
        else:
            pass

        # check os #
        if os.name == 'nt':
            if not is_use_pyknp:
                logger.warning(msg='It forces is_use_pyknp = True on Windows.')
            else:
                pass
            self.is_use_pyknp = True
        else:
            pass

        if is_use_pyknp or not server is None:
            self.juman = pyknp.Juman(command=command, server=server, port=port,
                                     timeout=self.timeout, rcfile=rcfile, option=option,
                                     pattern=pattern, **args)
            ### It overwrites juman_lines() method ###
            self.juman.juman_lines = self.__monkey_patch_juman_lines
        else:
            self.juman = JumanppHnadler(jumanpp_command=command,
                                        option=self.option,
                                        pattern=self.pattern,
                                        timeout_second=self.timeout)

    def __del__(self):
        if hasattr(self, "juman"):
            if isinstance(self.juman, JumanppHnadler):
                self.juman.stop_process()

    def __monkey_patch_juman_lines(self, input_str:str):
        """* What you can do
        - It overwrites juman_line() method because this method causes TypeError in python3
        """
        assert isinstance(self.juman, pyknp.Juman)
        if not self.juman.socket and not self.juman.subprocess:
            if self.juman.server is not None:
                self.juman.socket = MonkeyPatchSocket(self.juman.server, self.juman.port, b"RUN -e2\n")
            else:
                command = "%s %s" % (self.juman.command, self.juman.option)
                if self.juman.rcfile:
                    command += " -r %s" % self.juman.rcfile
                self.juman.subprocess = pyknp.Subprocess(command)
        if self.juman.socket:
            return self.juman.socket.query(input_str, pattern=self.juman.pattern)
        return self.juman.subprocess.query(input_str, pattern=self.juman.pattern)

    def call_juman_interface(self, input_str):
        """* What you can do
        - You call Juman tokenizer interface.

        * Output
        - pyknp.MList
        """
        # type: (str)->MList
        if isinstance(self.juman, pyknp.Juman):
            return self.juman.analysis(input_str)
        elif isinstance(self.juman, JumanppHnadler):
            result_analysis = self.juman.query(input_str)
            return MList(spec=result_analysis)

    def tokenize(self, sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (str, bool, bool, bool, bool, Callable[[str], str]) -> Union[TokenizedSenetence, List[str]]
        """
        :param sentence:
        :param ins_mecab:
        :param list_stopword:
        :param list_pos_candidate:
        :return:  list [tuple (unicode, unicode)]
        """
        assert isinstance(normalize, bool)
        assert isinstance(sentence, str)
        if normalize:
            normalized_sentence = func_normalizer(sentence)
        else:
            normalized_sentence = sentence

        result = self.call_juman_interface(normalized_sentence)
        token_objects = [
            juman_utils.extract_morphological_information(
                mrph_object=morph_object,
                is_surface=is_surface,
                is_feature=is_feature
            )
            for morph_object in result]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)

            return tokenized_objects

    def filter(self, parsed_sentence:TokenizedSenetence, pos_condition:List[Tuple[str, ...]]=None, stopwords:List[str]=None)->FilteredObject:
        assert isinstance(parsed_sentence, TokenizedSenetence)
        assert isinstance(pos_condition, (type(None), list))
        assert isinstance(stopwords, (type(None), list))

        return parsed_sentence.filter(pos_condition, stopwords)
<|endoftext|>


<commit_before># encoding: utf-8
"""
parse_match.py

Created by Thomas Mangin on 2015-06-22.
Copyright (c) 2009-2015 Exa Networks. All rights reserved.
"""

from exabgp.configuration.core import Section

from exabgp.configuration.flow.parser import source
from exabgp.configuration.flow.parser import destination
from exabgp.configuration.flow.parser import any_port
from exabgp.configuration.flow.parser import source_port
from exabgp.configuration.flow.parser import destination_port
from exabgp.configuration.flow.parser import tcp_flags
from exabgp.configuration.flow.parser import protocol
from exabgp.configuration.flow.parser import next_header
from exabgp.configuration.flow.parser import fragment
from exabgp.configuration.flow.parser import packet_length
from exabgp.configuration.flow.parser import icmp_code
from exabgp.configuration.flow.parser import icmp_type
from exabgp.configuration.flow.parser import dscp
from exabgp.configuration.flow.parser import traffic_class
from exabgp.configuration.flow.parser import flow_label

class ParseFlowMatch (Section):
	definition = [
		'source 10.0.0.0/24',
		'source ::1/128/0',
		'destination 10.0.1.0/24',
		'port 25',
		'source-port >1024',
		'destination-port =80|=3128|>8080&<8088',
		'packet-length >200&<300|>400&<500',
		'(ipv4 only) protocol [ udp tcp ]',
		'(ipv4 only) fragment [ not-a-fragment dont-fragment is-fragment first-fragment last-fragment ]',
		'(ipv6 only) next-header [ udp tcp ]',
		'(ipv6 only) flow-label >100&<2000',
	]

	syntax = \
		'match {\n' \
		'  %s;\n' \
		'}' % ';\n  '.join(definition)

	known = {
		'source':           source,
		'destination':      destination,
		'port':             any_port,
		'source-port':      source_port,
		'destination-port': destination_port,
		'protocol':         protocol,
		'packet-length':    packet_length,
		'tcp-flags':        tcp_flags,
		'next-header':      next_header,
		'fragment':         fragment,
		'icmp-code':        icmp_code,
		'icmp-type':        icmp_type,
		'packet-length':    packet_length,
		'dscp':             dscp,
		'traffic-class':    traffic_class,
		'flow-label':       flow_label,
	}

	# 'source-ipv4','destination-ipv4',

	action = {
		'source':           'nlri-add',
		'destination':      'nlri-add',
		'port':             'nlri-add',
		'source-port':      'nlri-add',
		'destination-port': 'nlri-add',
		'protocol':         'nlri-add',
		'packet-length':    'nlri-add',
		'tcp-flags':        'nlri-add',
		'next-header':      'nlri-add',
		'fragment':         'nlri-add',
		'icmp-code':        'nlri-add',
		'icmp-type':        'nlri-add',
		'packet-length':    'nlri-add',
		'dscp':             'nlri-add',
		'traffic-class':    'nlri-add',
		'flow-label':       'nlri-add',
	}

	name = 'flow/match'

	def __init__ (self, tokeniser, scope, error, logger):
		Section.__init__(self,tokeniser,scope,error,logger)

	def clear (self):
		pass

	def pre (self):
		self.scope.set(self.name,self.scope.get('flow/route'))
		return True

	def post (self):
		self.scope.pop(self.name)
		return True
<commit_msg>slight change to flow parsing code<commit_after># encoding: utf-8
"""
parse_match.py

Created by Thomas Mangin on 2015-06-22.
Copyright (c) 2009-2015 Exa Networks. All rights reserved.
"""

from exabgp.configuration.core import Section

from exabgp.configuration.flow.parser import source
from exabgp.configuration.flow.parser import destination
from exabgp.configuration.flow.parser import any_port
from exabgp.configuration.flow.parser import source_port
from exabgp.configuration.flow.parser import destination_port
from exabgp.configuration.flow.parser import tcp_flags
from exabgp.configuration.flow.parser import protocol
from exabgp.configuration.flow.parser import next_header
from exabgp.configuration.flow.parser import fragment
from exabgp.configuration.flow.parser import packet_length
from exabgp.configuration.flow.parser import icmp_code
from exabgp.configuration.flow.parser import icmp_type
from exabgp.configuration.flow.parser import dscp
from exabgp.configuration.flow.parser import traffic_class
from exabgp.configuration.flow.parser import flow_label

class ParseFlowMatch (Section):
	definition = [
		'source 10.0.0.0/24',
		'source ::1/128/0',
		'destination 10.0.1.0/24',
		'port 25',
		'source-port >1024',
		'destination-port [ =80 =3128 >8080&<8088 ]',
		'packet-length [ >200&<300 >400&<500 ]',
		'tcp-flags [ 0x20+0x8+0x1 #name-here ]  # to check',
		'(ipv4 only) protocol [ udp tcp ]',
		'(ipv4 only) fragment [ not-a-fragment dont-fragment is-fragment first-fragment last-fragment ]',
		'(ipv6 only) next-header [ udp tcp ]',
		'(ipv6 only) flow-label >100&<2000',
		'(ipv6 only) icmp-type 35  # to check',
		'(ipv6 only) icmp-code 55  # to check',
	]

	syntax = \
		'match {\n' \
		'  %s;\n' \
		'}' % ';\n  '.join(definition)

	known = {
		'source':           source,
		'destination':      destination,
		'source-ipv4':      source,
		'destination-ipv4': destination,
		'source-ipv6':      source,
		'destination-ipv6': destination,
		'port':             any_port,
		'source-port':      source_port,
		'destination-port': destination_port,
		'protocol':         protocol,
		'packet-length':    packet_length,
		'tcp-flags':        tcp_flags,
		'next-header':      next_header,
		'fragment':         fragment,
		'icmp-code':        icmp_code,
		'icmp-type':        icmp_type,
		'packet-length':    packet_length,
		'dscp':             dscp,
		'traffic-class':    traffic_class,
		'flow-label':       flow_label,
	}

	# 'source-ipv4','destination-ipv4',

	action = {
		'source':           'nlri-add',
		'destination':      'nlri-add',
		'port':             'nlri-add',
		'source-port':      'nlri-add',
		'destination-port': 'nlri-add',
		'protocol':         'nlri-add',
		'packet-length':    'nlri-add',
		'tcp-flags':        'nlri-add',
		'next-header':      'nlri-add',
		'fragment':         'nlri-add',
		'icmp-code':        'nlri-add',
		'icmp-type':        'nlri-add',
		'packet-length':    'nlri-add',
		'dscp':             'nlri-add',
		'traffic-class':    'nlri-add',
		'flow-label':       'nlri-add',
	}

	name = 'flow/match'

	def __init__ (self, tokeniser, scope, error, logger):
		Section.__init__(self,tokeniser,scope,error,logger)

	def clear (self):
		pass

	def pre (self):
		self.scope.set(self.name,self.scope.get('flow/route'))
		return True

	def post (self):
		self.scope.pop(self.name)
		return True
<|endoftext|>