<commit_before>
import roslib; roslib.load_manifest('hrl_pr2_kinematics')
import rospy

from equilibrium_point_control.msg import MechanismKinematicsRot
from equilibrium_point_control.msg import MechanismKinematicsJac

import epc
from threading import RLock

class Door_EPC(epc.EPC):

    def __init__(self, robot):
        epc.EPC.__init__(self, robot)

        self.mech_kinematics_lock = RLock()
        self.fit_circle_lock = RLock()

        rospy.Subscriber('mechanism_kinematics_rot',
                         MechanismKinematicsRot,
                         self.mechanism_kinematics_rot_cb)
        rospy.Subscriber('mechanism_kinematics_jac',
                         MechanismKinematicsJac,
                         self.mechanism_kinematics_jac_cb)
        self.mech_traj_pub = rospy.Publisher('mechanism_trajectory', Point32)
        self.eq_pt_not_moving_counter = 0

    ## log the joint angles, equi pt joint angles and forces.
    def log_state(self, arm):
        t_now = rospy.get_time()
        q_now = self.robot.get_joint_angles(arm)

        self.pull_trajectory.q_list.append(q_now)
        self.pull_trajectory.time_list.append(t_now)

        self.eq_pt_trajectory.q_list.append(self.q_guess) # see equi_pt_generator - q_guess is the config for the eq point.
        self.eq_pt_trajectory.time_list.append(t_now)

        wrist_force = self.robot.get_wrist_force(arm, base_frame=True)
        self.force_trajectory.f_list.append(wrist_force.A1.tolist())
        self.force_trajectory.time_list.append(t_now)
        return '' # log_state also used as a rapid_call_func

    ## constantly update the estimate of the kinematics and move the
    # equilibrium point along the tangent of the estimated arc, and
    # try to keep the radial force constant.
    # @param h_force_possible - True (hook side) or False (hook up).
    # @param v_force_possible - False (hook side) or True (hook up).
    # Is maintaining a radial force possible or not (based on hook
    # geometry and orientation)
    # @param cep_vel - tangential velocity of the cep in m/s
    def equi_pt_generator_control_radial_force(self, arm, rot_mat,
                           h_force_possible, v_force_possible, cep_vel):
        self.log_state(arm)
        step_size = 0.1 * cep_vel # 0.1 is the time interval between calls to the equi_generator function (see pull)
        q_eq = self.update_eq_point(arm, self.eq_motion_vec, step_size,
                                    rot_mat)

        stop = self.common_stopping_conditions()

        wrist_force = self.robot.get_wrist_force(arm, base_frame=True)
        mag = np.linalg.norm(wrist_force)

        curr_pos = self.robot.FK(arm,self.pull_trajectory.q_list[-1])
        if len(self.pull_trajectory.q_list)>1:
            start_pos = np.matrix(self.cartesian_pts_list[0]).T
        else:
            start_pos = curr_pos

        #mechanism kinematics.
        self.mech_traj_pub.publish(Point32(curr_pos[0,0],
                                        curr_pos[1,0], curr_pos[2,0]))

        if self.use_jacobian:
            self.mech_kinematics_lock.acquire()

            self.cartesian_pts_list.append(curr_pos.A1.tolist())
            tangential_vec_ts = self.tangential_vec_ts
            force_vec_ts = self.constraint_vec_1_ts + self.constraint_vec_2_ts

            # this is specifically for the right arm, and lots of
            # other assumptions about the hook etc. (Advait, Feb 28, 2010)
            if h_force_possible:
                force_vec_ts[2,0] = 0.
            if v_force_possible:
                force_vec_ts[1,0] = 0.
            force_vec_ts = force_vec_ts / np.linalg.norm(force_vec_ts)
            if force_vec_ts[2,0] < 0.: # only allowing upward force.
                force_vec_ts = -force_vec_ts
            if force_vec_ts[1,0] < 0.: # only allowing force to the left
                force_vec_ts = -force_vec_ts

            self.mech_kinematics_lock.release()

            force_vec = force_vec_ts
            tangential_vec = tangential_vec_ts

        if self.use_rotation_center:
            self.fit_circle_lock.acquire()
            self.cartesian_pts_list.append(curr_pos.A1.tolist())
            rad = self.rad
            cx_start, cy_start = self.cx_start, self.cy_start
            cz_start = self.cz_start
            self.fit_circle_lock.release()
            cx, cy = cx_start, cy_start
            cz = cz_start

            radial_vec = curr_pos_tl-np.matrix([cx,cy,cz]).T
            radial_vec = radial_vec/np.linalg.norm(radial_vec)
            if cy_start<start_pos[1,0]:
                tan_x,tan_y = -radial_vec[1,0],radial_vec[0,0]
            else:
                tan_x,tan_y = radial_vec[1,0],-radial_vec[0,0]

            if tan_x > 0. and (start_pos[0,0]-curr_pos[0,0]) < 0.09:
                tan_x = -tan_x
                tan_y = -tan_y

            if cy_start > start_pos[1,0]:
                radial_vec = -radial_vec # axis to the left, want force in
                                       # anti-radial direction.
            rv = radial_vec
            force_vec = np.matrix([rv[0,0], rv[1,0], 0.]).T
            tangential_vec = np.matrix([tan_x, tan_y, 0.]).T

            tangential_vec_ts = tangential_vec
            radial_vec_ts = radial_vec
            force_vec_ts = force_vec

        f_vec = -1*np.array([wrist_force[0,0], wrist_force[1,0],
                             wrist_force[2,0]])
        f_rad_mag = np.dot(f_vec, force_vec.A1)
        err = f_rad_mag-5.
        if err>0.:
            kp = -0.1
        else:
            kp = -0.2
        radial_motion_mag = kp * err # radial_motion_mag in cm (depends on eq_motion step size)

        if self.use_rotation_center:
            if h_force_possible == False and parallel_to_floor:
                radial_motion_mag = 0.
            if v_force_possible == False and parallel_to_floor == False:
                radial_motion_mag = 0.

        radial_motion_vec =  force_vec * radial_motion_mag
        self.eq_motion_vec = copy.copy(tangential_vec)
        self.eq_motion_vec += radial_motion_vec

        self.prev_force_mag = mag

        if self.init_tangent_vector == None:
            self.init_tangent_vector = copy.copy(tangential_vec_ts)
        c = np.dot(tangential_vec_ts.A1, self.init_tangent_vector.A1)
        ang = np.arccos(c)

        dist_moved = np.dot((curr_pos - start_pos).A1, self.tangential_vec_ts.A1)
        ftan = abs(np.dot(wrist_force.A1, tangential_vec.A1))
        if abs(dist_moved) > 0.09 and self.hooked_location_moved == False:
            # change the force threshold once the hook has started pulling.
            self.hooked_location_moved = True
            self.eq_force_threshold = ut.bound(mag+30.,20.,80.)
            self.ftan_threshold = self.ftan_threshold + max(10., 1.5*ftan)
        if self.hooked_location_moved:
            if abs(tangential_vec_ts[2,0]) < 0.2 and ftan > self.ftan_threshold:
                stop = 'ftan threshold exceed: %f'%ftan
        else:
            self.ftan_threshold = max(self.ftan_threshold, ftan)

        if self.hooked_location_moved and ang > math.radians(90.):
            print 'Angle:', math.degrees(ang)
            self.open_ang_exceed_count += 1
            if self.open_ang_exceed_count > 2:
                stop = 'opened mechanism through large angle: %.1f'%(math.degrees(ang))
        else:
            self.open_ang_exceed_count = 0

        self.mech_time_list.append(time.time())
        self.mech_x_list.append(ang)
        self.f_rad_list.append(f_rad_mag)
        self.f_tan_list.append(np.dot(f_vec, tangential_vec.A1))
        self.tan_vec_list.append(tangential_vec_ts.A1.tolist())
        self.rad_vec_list.append(force_vec_ts.A1.tolist())

        return stop, q_eq

    def pull(self, arm, hook_angle, force_threshold, ea,
             kinematics_estimation = 'rotation_center', pull_left = False):

        self.init_tangent_vector = None
        self.open_ang_exceed_count = 0.
        if kinematics_estimation == 'rotation_center':
            self.use_rotation_center = True
        else:
            self.use_rotation_center = False

        if kinematics_estimation == 'jacobian':
            self.use_jacobian = True
        else:
            self.use_jacobian = False

        #rot_mat = tr.Rz(hook_angle)*tr.Ry(math.radians(-90))
        rot_mat = rot_mat_from_angles(hook_angle, surface_angle)

        time_dict = {}
        time_dict['before_pull'] = time.time()

        stiffness_scale = self.robot.arm_settings[arm].stiffness_scale
        self.pull_trajectory = at.JointTrajectory()
        self.jt_torque_trajectory = at.JointTrajectory()
        self.eq_pt_trajectory = at.JointTrajectory()
        self.force_trajectory = at.ForceTrajectory()

        self.robot.step()
        start_config = self.robot.get_joint_angles(arm)

        self.eq_force_threshold = force_threshold
        self.ftan_threshold = 2.
        self.hooked_location_moved = False # flag to indicate when the hooking location started moving.
        self.prev_force_mag = np.linalg.norm(self.robot.get_wrist_force(arm))
        self.eq_motion_vec = np.matrix([-1.,0.,0.]).T # might want to change this to account for the surface_angle.
        self.slip_count = 0

        self.eq_pt_cartesian = self.robot.FK(arm, ea)
        self.eq_pt_cartesian_ts = self.robot.FK(arm, ea)
        self.start_pos = copy.copy(self.eq_pt_cartesian)
        self.q_guess = ea

        if not pull_left:
            self.tangential_vec_ts = np.matrix([-1.,0.,0.]).T
            self.constraint_vec_2_ts = np.matrix([0.,0.,1.]).T
            self.constraint_vec_1_ts = np.matrix([0.,1.,0.]).T
        else:
            self.tangential_vec_ts = np.matrix([0.,1.,0.]).T
            self.constraint_vec_2_ts = np.matrix([0.,0.,1.]).T
            self.constraint_vec_1_ts = np.matrix([1.,0.,0.]).T

        self.mech_time_list = []
        self.mech_x_list = []
        self.f_rad_list = []
        self.f_tan_list = []
        self.tan_vec_list = []
        self.rad_vec_list = []

        self.cartesian_pts_list = []
        ee_pos = self.robot.end_effector_pos(arm)

        if self.use_rotation_center:
        # this might have to change depending on left and right
        # arm? or maybe not since the right arm can open both
        # doors.
            self.cx_start = ee_pos[0,0]
            self.cy_start = ee_pos[1,0]-1.0
            self.cz_start = ee_pos[2,0]
            self.rad = 5.0

        h_force_possible = abs(hook_angle) < math.radians(30.)
        v_force_possible = abs(hook_angle) > math.radians(60.)
        arg_list = [arm, rot_mat, h_force_possible, v_force_possible, cep_vel]
        result, jep = self.epc_motion(self.eqpt_gen_control_radial_force,
                                            0.1, arm, arg_list, self.log_state)

        print 'EPC motion result:', result
        print 'Original force threshold:', force_threshold
        print 'Adapted force threshold:', self.eq_force_threshold
        print 'Adapted ftan threshold:', self.ftan_threshold

        time_dict['after_pull'] = time.time()

        d = {'actual': self.pull_trajectory, 'eq_pt': self.eq_pt_trajectory,
             'force': self.force_trajectory, 'torque': self.jt_torque_trajectory,
             'info': info_string, 'force_threshold': force_threshold,
             'eq_force_threshold': self.eq_force_threshold, 'hook_angle':hook_angle,
             'result':result, 'time_dict':time_dict,
             'cep_vel': cep_vel,
             'ftan_threshold': self.ftan_threshold}

        self.robot.step()

        ut.save_pickle(d,'pull_trajectories_'+d['info']+'_'+ut.formatted_time()+'.pkl')

        dd = {'mechanism_x': self.mech_x_list,
              'mechanism_time': self.mech_time_list,
              'force_rad_list': self.f_rad_list,
              'force_tan_list': self.f_tan_list,
              'tan_vec_list': self.tan_vec_list,
              'rad_vec_list': self.rad_vec_list
              }
        ut.save_pickle(dd,'mechanism_trajectories_robot_'+d['info']+'_'+ut.formatted_time()+'.pkl')

if __name__ == '__main__':

    print 'Hello World'

<commit_msg>M    hrl_pr2_kinematics_tutorials/door_epc.py copied and modified a large portion of door opening code with stationary base. Yet to add initial hook on to the mechanism.<commit_after>
import roslib; roslib.load_manifest('hrl_pr2_kinematics')
import rospy

from equilibrium_point_control.msg import MechanismKinematicsRot
from equilibrium_point_control.msg import MechanismKinematicsJac

import epc
from threading import RLock

class Door_EPC(epc.EPC):

    def __init__(self, robot):
        epc.EPC.__init__(self, robot)

        self.mech_kinematics_lock = RLock()
        self.fit_circle_lock = RLock()

        rospy.Subscriber('mechanism_kinematics_rot',
                         MechanismKinematicsRot,
                         self.mechanism_kinematics_rot_cb)
        rospy.Subscriber('mechanism_kinematics_jac',
                         MechanismKinematicsJac,
                         self.mechanism_kinematics_jac_cb)
        self.mech_traj_pub = rospy.Publisher('mechanism_trajectory', Point32)
        self.eq_pt_not_moving_counter = 0

    ## log the joint angles, equi pt joint angles and forces.
    def log_state(self, arm):
        t_now = rospy.get_time()
        q_now = self.robot.get_joint_angles(arm)

        self.pull_trajectory.q_list.append(q_now)
        self.pull_trajectory.time_list.append(t_now)

        self.eq_pt_trajectory.q_list.append(self.q_guess) # see equi_pt_generator - q_guess is the config for the eq point.
        self.eq_pt_trajectory.time_list.append(t_now)

        wrist_force = self.robot.get_wrist_force(arm, base_frame=True)
        self.force_trajectory.f_list.append(wrist_force.A1.tolist())
        self.force_trajectory.time_list.append(t_now)
        return '' # log_state also used as a rapid_call_func

    ##
    # @param arm - 'right_arm' or 'left_arm'
    # @param motion vec is in tl frame.
    # @param step_size - distance (meters) through which CEP should move
    # @param rot_mat - rotation matrix for IK
    # @return JEP
    def update_eq_point(self, arm, motion_vec, step_size, rot_mat):
        self.eq_pt_cartesian = self.eq_pt_cartesian_ts
        next_pt = self.eq_pt_cartesian + step_size * motion_vec
        q_eq = self.robot.IK(arm, next_pt, rot_mat, self.q_guess)
        self.eq_pt_cartesian = next_pt
        self.eq_pt_cartesian_ts = self.eq_pt_cartesian
        self.q_guess = q_eq
        return q_eq

    def common_stopping_conditions(self):
        stop = ''
        if self.q_guess == None:
            stop = 'IK fail'

        wrist_force = self.robot.get_wrist_force('right_arm',base_frame=True)
        mag = np.linalg.norm(wrist_force)
        print 'force magnitude:', mag
        if mag > self.eq_force_threshold:
            stop = 'force exceed'

        if mag < 1.2 and self.hooked_location_moved:
            if (self.prev_force_mag - mag) > 30.:
                stop = 'slip: force step decrease and below thresold.'
                #stop = ''
            else:
                self.slip_count += 1
        else:
            self.slip_count = 0

        if self.slip_count == 10:
            stop = 'slip: force below threshold for too long.'
        return stop

    ## constantly update the estimate of the kinematics and move the
    # equilibrium point along the tangent of the estimated arc, and
    # try to keep the radial force constant.
    # @param h_force_possible - True (hook side) or False (hook up).
    # @param v_force_possible - False (hook side) or True (hook up).
    # Is maintaining a radial force possible or not (based on hook
    # geometry and orientation)
    # @param cep_vel - tangential velocity of the cep in m/s
    def equi_pt_generator_control_radial_force(self, arm, rot_mat,
                           h_force_possible, v_force_possible, cep_vel):
        self.log_state(arm)
        step_size = 0.1 * cep_vel # 0.1 is the time interval between calls to the equi_generator function (see pull)
        q_eq = self.update_eq_point(arm, self.eq_motion_vec, step_size,
                                    rot_mat)
        stop = self.common_stopping_conditions()

        wrist_force = self.robot.get_wrist_force(arm, base_frame=True)
        mag = np.linalg.norm(wrist_force)

        curr_pos = self.robot.FK(arm,self.pull_trajectory.q_list[-1])
        if len(self.pull_trajectory.q_list)>1:
            start_pos = np.matrix(self.cartesian_pts_list[0]).T
        else:
            start_pos = curr_pos

        #mechanism kinematics.
        self.mech_traj_pub.publish(Point32(curr_pos[0,0],
                                        curr_pos[1,0], curr_pos[2,0]))

        if self.use_jacobian:
            self.mech_kinematics_lock.acquire()

            self.cartesian_pts_list.append(curr_pos.A1.tolist())
            tangential_vec_ts = self.tangential_vec_ts
            force_vec_ts = self.constraint_vec_1_ts + self.constraint_vec_2_ts

            # this is specifically for the right arm, and lots of
            # other assumptions about the hook etc. (Advait, Feb 28, 2010)
            if h_force_possible:
                force_vec_ts[2,0] = 0.
            if v_force_possible:
                force_vec_ts[1,0] = 0.
            force_vec_ts = force_vec_ts / np.linalg.norm(force_vec_ts)
            if force_vec_ts[2,0] < 0.: # only allowing upward force.
                force_vec_ts = -force_vec_ts
            if force_vec_ts[1,0] < 0.: # only allowing force to the left
                force_vec_ts = -force_vec_ts

            self.mech_kinematics_lock.release()

            force_vec = force_vec_ts
            tangential_vec = tangential_vec_ts

        if self.use_rotation_center:
            self.fit_circle_lock.acquire()
            self.cartesian_pts_list.append(curr_pos.A1.tolist())
            rad = self.rad
            cx_start, cy_start = self.cx_start, self.cy_start
            cz_start = self.cz_start
            self.fit_circle_lock.release()
            cx, cy = cx_start, cy_start
            cz = cz_start

            radial_vec = curr_pos_tl-np.matrix([cx,cy,cz]).T
            radial_vec = radial_vec/np.linalg.norm(radial_vec)
            if cy_start<start_pos[1,0]:
                tan_x,tan_y = -radial_vec[1,0],radial_vec[0,0]
            else:
                tan_x,tan_y = radial_vec[1,0],-radial_vec[0,0]

            if tan_x > 0. and (start_pos[0,0]-curr_pos[0,0]) < 0.09:
                tan_x = -tan_x
                tan_y = -tan_y

            if cy_start > start_pos[1,0]:
                radial_vec = -radial_vec # axis to the left, want force in
                                       # anti-radial direction.
            rv = radial_vec
            force_vec = np.matrix([rv[0,0], rv[1,0], 0.]).T
            tangential_vec = np.matrix([tan_x, tan_y, 0.]).T

            tangential_vec_ts = tangential_vec
            radial_vec_ts = radial_vec
            force_vec_ts = force_vec

        f_vec = -1*np.array([wrist_force[0,0], wrist_force[1,0],
                             wrist_force[2,0]])
        f_rad_mag = np.dot(f_vec, force_vec.A1)
        err = f_rad_mag-5.
        if err>0.:
            kp = -0.1
        else:
            kp = -0.2
        radial_motion_mag = kp * err # radial_motion_mag in cm (depends on eq_motion step size)

        if self.use_rotation_center:
            if h_force_possible == False and parallel_to_floor:
                radial_motion_mag = 0.
            if v_force_possible == False and parallel_to_floor == False:
                radial_motion_mag = 0.

        radial_motion_vec =  force_vec * radial_motion_mag
        self.eq_motion_vec = copy.copy(tangential_vec)
        self.eq_motion_vec += radial_motion_vec

        self.prev_force_mag = mag

        if self.init_tangent_vector == None:
            self.init_tangent_vector = copy.copy(tangential_vec_ts)
        c = np.dot(tangential_vec_ts.A1, self.init_tangent_vector.A1)
        ang = np.arccos(c)

        dist_moved = np.dot((curr_pos - start_pos).A1, self.tangential_vec_ts.A1)
        ftan = abs(np.dot(wrist_force.A1, tangential_vec.A1))
        if abs(dist_moved) > 0.09 and self.hooked_location_moved == False:
            # change the force threshold once the hook has started pulling.
            self.hooked_location_moved = True
            self.eq_force_threshold = ut.bound(mag+30.,20.,80.)
            self.ftan_threshold = self.ftan_threshold + max(10., 1.5*ftan)
        if self.hooked_location_moved:
            if abs(tangential_vec_ts[2,0]) < 0.2 and ftan > self.ftan_threshold:
                stop = 'ftan threshold exceed: %f'%ftan
        else:
            self.ftan_threshold = max(self.ftan_threshold, ftan)

        if self.hooked_location_moved and ang > math.radians(90.):
            print 'Angle:', math.degrees(ang)
            self.open_ang_exceed_count += 1
            if self.open_ang_exceed_count > 2:
                stop = 'opened mechanism through large angle: %.1f'%(math.degrees(ang))
        else:
            self.open_ang_exceed_count = 0

        self.mech_time_list.append(time.time())
        self.mech_x_list.append(ang)
        self.f_rad_list.append(f_rad_mag)
        self.f_tan_list.append(np.dot(f_vec, tangential_vec.A1))
        self.tan_vec_list.append(tangential_vec_ts.A1.tolist())
        self.rad_vec_list.append(force_vec_ts.A1.tolist())

        return stop, q_eq

    def pull(self, arm, hook_angle, force_threshold, ea,
             kinematics_estimation = 'rotation_center', pull_left = False):

        self.init_tangent_vector = None
        self.open_ang_exceed_count = 0.
        if kinematics_estimation == 'rotation_center':
            self.use_rotation_center = True
        else:
            self.use_rotation_center = False

        if kinematics_estimation == 'jacobian':
            self.use_jacobian = True
        else:
            self.use_jacobian = False

        #rot_mat = tr.Rz(hook_angle)*tr.Ry(math.radians(-90))
        rot_mat = rot_mat_from_angles(hook_angle, surface_angle)

        time_dict = {}
        time_dict['before_pull'] = time.time()

        stiffness_scale = self.robot.arm_settings[arm].stiffness_scale
        self.pull_trajectory = at.JointTrajectory()
        self.jt_torque_trajectory = at.JointTrajectory()
        self.eq_pt_trajectory = at.JointTrajectory()
        self.force_trajectory = at.ForceTrajectory()

        self.robot.step()
        start_config = self.robot.get_joint_angles(arm)

        self.eq_force_threshold = force_threshold
        self.ftan_threshold = 2.
        self.hooked_location_moved = False # flag to indicate when the hooking location started moving.
        self.prev_force_mag = np.linalg.norm(self.robot.get_wrist_force(arm))
        self.eq_motion_vec = np.matrix([-1.,0.,0.]).T # might want to change this to account for the surface_angle.
        self.slip_count = 0

        self.eq_pt_cartesian = self.robot.FK(arm, ea)
        self.eq_pt_cartesian_ts = self.robot.FK(arm, ea)
        self.start_pos = copy.copy(self.eq_pt_cartesian)
        self.q_guess = ea

        if not pull_left:
            self.tangential_vec_ts = np.matrix([-1.,0.,0.]).T
            self.constraint_vec_2_ts = np.matrix([0.,0.,1.]).T
            self.constraint_vec_1_ts = np.matrix([0.,1.,0.]).T
        else:
            self.tangential_vec_ts = np.matrix([0.,1.,0.]).T
            self.constraint_vec_2_ts = np.matrix([0.,0.,1.]).T
            self.constraint_vec_1_ts = np.matrix([1.,0.,0.]).T

        self.mech_time_list = []
        self.mech_x_list = []
        self.f_rad_list = []
        self.f_tan_list = []
        self.tan_vec_list = []
        self.rad_vec_list = []

        self.cartesian_pts_list = []
        ee_pos = self.robot.end_effector_pos(arm)

        if self.use_rotation_center:
        # this might have to change depending on left and right
        # arm? or maybe not since the right arm can open both
        # doors.
            self.cx_start = ee_pos[0,0]
            self.cy_start = ee_pos[1,0]-1.0
            self.cz_start = ee_pos[2,0]
            self.rad = 5.0

        h_force_possible = abs(hook_angle) < math.radians(30.)
        v_force_possible = abs(hook_angle) > math.radians(60.)
        arg_list = [arm, rot_mat, h_force_possible, v_force_possible, cep_vel]
        result, jep = self.epc_motion(self.eqpt_gen_control_radial_force,
                                            0.1, arm, arg_list, self.log_state)

        print 'EPC motion result:', result
        print 'Original force threshold:', force_threshold
        print 'Adapted force threshold:', self.eq_force_threshold
        print 'Adapted ftan threshold:', self.ftan_threshold

        time_dict['after_pull'] = time.time()

        d = {'actual': self.pull_trajectory, 'eq_pt': self.eq_pt_trajectory,
             'force': self.force_trajectory, 'torque': self.jt_torque_trajectory,
             'info': info_string, 'force_threshold': force_threshold,
             'eq_force_threshold': self.eq_force_threshold, 'hook_angle':hook_angle,
             'result':result, 'time_dict':time_dict,
             'cep_vel': cep_vel,
             'ftan_threshold': self.ftan_threshold}

        self.robot.step()

        ut.save_pickle(d,'pull_trajectories_'+d['info']+'_'+ut.formatted_time()+'.pkl')

        dd = {'mechanism_x': self.mech_x_list,
              'mechanism_time': self.mech_time_list,
              'force_rad_list': self.f_rad_list,
              'force_tan_list': self.f_tan_list,
              'tan_vec_list': self.tan_vec_list,
              'rad_vec_list': self.rad_vec_list
              }
        ut.save_pickle(dd,'mechanism_trajectories_robot_'+d['info']+'_'+ut.formatted_time()+'.pkl')

if __name__ == '__main__':

    print 'Hello World'

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import unittest

import _epitran

class TestNormalizePunc(unittest.TestCase):
    def setUp(self):
        self.epi = _epitran.Epitran(u'vie-Latn')

    def test_tieng(self):
        tr = self.epi.transliterate('tiếng')
        print(tr)
        self.assertEqual(tr, 'tiə̯ŋ')

    def test_viet(self):
        tr = self.epi.transliterate('Việt')
        print(tr)
        self.assertEqual(tr, 'viə̯t')

    def test_ngang(self):
        tr = self.epi.transliterate('ngang')
        print(tr)
        self.assertEqual(tr, 'ŋaːŋ')

    def test_huyen(self):
        tr = self.epi.transliterate('huyền')
        print(tr)
        self.assertEqual(tr, 'hwiə̯n')

    def test_sac(self):
        tr = self.epi.transliterate('sắc')
        print(tr)
        self.assertEqual(tr, 'sak')

    def test_hoi(self):
        tr = self.epi.transliterate('hỏi')
        print(tr)
        self.assertEqual(tr, 'hoj')

    def test_nga(self):
        tr = self.epi.transliterate('ngã')
        print(tr)
        self.assertEqual(tr, 'ŋɑː')

    def test_nang(self):
        tr = self.epi.transliterate('nặng')
        print(tr)
        self.assertEqual(, 'naŋ')
<commit_msg>Corrected vietnamese tests<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import unittest

import _epitran

class TestNormalizePunc(unittest.TestCase):
    def setUp(self):
        self.epi = _epitran.Epitran(u'vie-Latn')

    def test_tieng(self):
        tr = self.epi.transliterate('tiếng')
        self.assertEqual(tr, 'tiə̯ŋ')

    def test_viet(self):
        tr = self.epi.transliterate('Việt')
        self.assertEqual(tr, 'viə̯t')

    def test_ngang(self):
        tr = self.epi.transliterate('ngang')
        self.assertEqual(tr, 'ŋaːŋ')

    def test_huyen(self):
        tr = self.epi.transliterate('huyền')
        self.assertEqual(tr, 'hwiə̯n')

    def test_sac(self):
        tr = self.epi.transliterate('sắc')
        self.assertEqual(tr, 'sak')

    def test_hoi(self):
        tr = self.epi.transliterate('hỏi')
        self.assertEqual(tr, 'hɔj')

    def test_nga(self):
        tr = self.epi.transliterate('ngã')
        self.assertEqual(tr, 'ŋaː')

    def test_nang(self):
        tr = self.epi.transliterate('nặng')
        self.assertEqual(tr, 'naŋ')
<|endoftext|>


<commit_before># This module should be imported from REPL, not run from command line.
import socket
import uos
import websocket
import websocket_helper

listen_s = None
client_s = None

def wait_connection():
    global listen_s, client_s
    listen_s = socket.socket()
    listen_s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    ai = socket.getaddrinfo("0.0.0.0", 8266)
    print("Bind address info:", ai)
    addr = ai[0][4]

    listen_s.bind(addr)
    listen_s.listen(1)
    client_s, remote_addr = listen_s.accept()
    print(client_s)
    return client_s

def start():
    global listen_s, client_s
    uos.dupterm(None)
    if client_s:
        client_s.close()
    if listen_s:
        listen_s.close()
    cl = wait_connection()
    websocket_helper.server_handshake(cl)
    ws = websocket.websocket(cl, True)
    cl.setblocking(False)
    # notify REPL on socket incoming data
    cl.setsockopt(socket.SOL_SOCKET, 20, uos.dupterm_notify)
    uos.dupterm(ws)
    print("Connected")

start()
<commit_msg>esp8266/scripts/webrepl: Convert to persistent daemon.<commit_after># This module should be imported from REPL, not run from command line.
import socket
import uos
import websocket
import websocket_helper

listen_s = None
client_s = None

def setup_conn():
    global listen_s, client_s
    listen_s = socket.socket()
    listen_s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    ai = socket.getaddrinfo("0.0.0.0", 8266)
    print("Bind address info:", ai)
    addr = ai[0][4]

    listen_s.bind(addr)
    listen_s.listen(1)
    listen_s.setsockopt(socket.SOL_SOCKET, 20, accept_conn)
    print("WebREPL daemon started on port 8266")

def accept_conn(listen_sock):
    global client_s
    cl, remote_addr = listen_sock.accept()
    client_s = cl
    websocket_helper.server_handshake(cl)
    ws = websocket.websocket(cl, True)
    cl.setblocking(False)
    # notify REPL on socket incoming data
    cl.setsockopt(socket.SOL_SOCKET, 20, uos.dupterm_notify)
    uos.dupterm(ws)
    print("WebREPL connected\n>>> ", end="")

def stop():
    global listen_s, client_s
    uos.dupterm(None)
    if client_s:
        client_s.close()
    if listen_s:
        listen_s.close()

def start():
    stop()
    setup_conn()

start()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2016-2018  Flensburg University of Applied Sciences,
# Europa-Universität Flensburg,
# Centre for Sustainable Energy Systems

# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# File description for read-the-docs
""" This module contains functions for calculating representative days/weeks
based on a pyPSA network object. It is designed to be used for the `lopf`
method. Essentially the tsam package
( https://github.com/FZJ-IEK3-VSA/tsam ), which is developed by
Leander Kotzur is used.

Remaining questions/tasks:

- Does it makes sense to cluster normed values?
- Include scaling method for yearly sums
"""

import pandas as pd
import os
if 'READTHEDOCS' not in os.environ:
    import pyomo.environ as po
    import tsam.timeseriesaggregation as tsam

__copyright__ = ("Flensburg University of Applied Sciences, "
                 "Europa-Universität Flensburg, "
                 "Centre for Sustainable Energy Systems")
__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"
__author__ = "Simon Hilpert"

def snapshot_clustering(network, how='daily', clusters=10):
    """
    """

    network = run(network=network.copy(), n_clusters=clusters,
                  how=how, normed=False)

    return network

def tsam_cluster(timeseries_df,
                 typical_periods=10,
                 how='daily',
                 extremePeriodMethod = 'None'):
    """
    Parameters
    ----------
    df : pd.DataFrame
        DataFrame with timeseries to cluster
    extremePeriodMethod: {'None','append','new_cluster_center',
                           'replace_cluster_center'}, default: 'None'
        Method how to integrate extreme Periods
        into to the typical period profiles.
        None: No integration at all.
        'append': append typical Periods to cluster centers
        'new_cluster_center': add the extreme period as additional cluster
            center. It is checked then for all Periods if they fit better
            to the this new center or their original cluster center.
        'replace_cluster_center': replaces the cluster center of the
            cluster where the extreme period belongs to with the periodly
            profile of the extreme period. (Worst case system design)

    Returns
    -------
    timeseries : pd.DataFrame
        Clustered timeseries
    """

    if how == 'daily':
        hours = 24
        period = ' days'
    if how == 'weekly':
        hours = 168
        period = ' weeks'

    print('Snapshot clustering to ' + str(typical_periods) + period +
          ' using extreme period method: ' + extremePeriodMethod)

    aggregation = tsam.TimeSeriesAggregation(
        timeseries_df,
        noTypicalPeriods=typical_periods,
        extremePeriodMethod = extremePeriodMethod,
        addPeakMin = ['residual_load'],
        addPeakMax = ['residual_load'],
        rescaleClusterPeriods=False,
        hoursPerPeriod=hours,
        clusterMethod='hierarchical')

    timeseries = aggregation.createTypicalPeriods()
    cluster_weights = aggregation.clusterPeriodNoOccur
    clusterOrder =aggregation.clusterOrder
    clusterCenterIndices= aggregation.clusterCenterIndices

    if extremePeriodMethod  != 'None':
        for i in aggregation.extremePeriods.keys():
            clusterCenterIndices.insert(
                    aggregation.extremePeriods[i]['newClusterNo'],
                    aggregation.extremePeriods[i]['stepNo'])

    # get all index for every hour of that day of the clusterCenterIndices
    start = []
    # get the first hour of the clusterCenterIndices (days start with 0)
    for i in clusterCenterIndices:
        start.append(i * hours)

    # get a list with all hours belonging to the clusterCenterIndices
    nrhours = []
    for j in start:
        nrhours.append(j)
        x = 1
        while x < hours:
            j = j + 1
            nrhours.append(j)
            x = x + 1

    # get the origial Datetimeindex
    dates = timeseries_df.iloc[nrhours].index

    #get list of representative days
    representative_day=[]

    #cluster:medoid des jeweiligen Clusters
    dic_clusterCenterIndices = dict(enumerate(clusterCenterIndices))
    for i in clusterOrder:
        representative_day.append(dic_clusterCenterIndices[i])

    #get list of last hour of representative days
    last_hour_datetime=[]
    for i in representative_day:
        last_hour = i * hours + hours - 1
        last_hour_datetime.append(timeseries_df.index[last_hour])

    #create a dataframe (index=nr. of day in a year/candidate)
    df_cluster =  pd.DataFrame({
                        'Cluster': clusterOrder, #Cluster of the day
                        'RepresentativeDay': representative_day, #representative day of the cluster
                        'last_hour_RepresentativeDay': last_hour_datetime}) #last hour of the cluster
    df_cluster.index = df_cluster.index + 1
    df_cluster.index.name = 'Candidate'

    #create a dataframe each timeseries (h) and its candiddate day (i) df_i_h
    nr_day = []
    x = len(timeseries_df.index)/hours+1

    for i in range(1,int(x)):
        j=1
        while j <= hours:
            nr_day.append(i)
            j=j+1
    df_i_h = pd.DataFrame({'Timeseries': timeseries_df.index,
                        'Candidate_day': nr_day})
    df_i_h.set_index('Timeseries',inplace=True)

    return df_cluster, cluster_weights, dates, hours, df_i_h

def run(network, n_clusters=None, how='daily',
        normed=False):
    """
    """

    # calculate clusters
    df_cluster, cluster_weights, dates, hours, df_i_h= tsam_cluster(
                prepare_pypsa_timeseries(network),
                typical_periods=n_clusters,
                how='daily',
                extremePeriodMethod = 'None')
    network.cluster = df_cluster
    network.cluster_ts = df_i_h

    update_data_frames(network, cluster_weights, dates, hours)

    return network

def prepare_pypsa_timeseries(network, normed=False):
    """
    """
    if normed:
        normed_loads = network.loads_t.p_set / network.loads_t.p_set.max()
        normed_loads.columns = 'L' + normed_loads.columns
        normed_renewables = network.generators_t.p_max_pu
        normed_renewables.columns = 'G' + normed_renewables.columns

        df = pd.concat([normed_renewables,
                        normed_loads], axis=1)
    else:
        loads = network.loads_t.p_set.copy()
        loads.columns = 'L' + loads.columns
        renewables = network.generators_t.p_max_pu.mul(
                network.generators.p_nom[
                network.generators_t.p_max_pu.columns], axis = 1).copy()
        renewables.columns = 'G' + renewables.columns
        residual_load=pd.DataFrame()
        residual_load['residual_load']=loads.sum(axis=1)-renewables.sum(axis=1)
        df = pd.concat([renewables, loads, residual_load], axis=1)

    return df

def update_data_frames(network, cluster_weights, dates, hours):
    """ Updates the snapshots, snapshots weights and the dataframes based on
    the original data in the network and the medoids created by clustering
    these original data.

    Parameters
    -----------
    network : pyPSA network object
    cluster_weights: dictionary
    dates: Datetimeindex

    Returns
    -------
    network

    """
    network.snapshot_weightings = network.snapshot_weightings.loc[dates]
    network.snapshots = network.snapshot_weightings.index

    # set new snapshot weights from cluster_weights
    snapshot_weightings = []
    for i in cluster_weights.values():
        x = 0
        while x < hours:
            snapshot_weightings.append(i)
            x += 1
    for i in range(len(network.snapshot_weightings)):
        network.snapshot_weightings[i] = snapshot_weightings[i]

    # put the snapshot in the right order
    network.snapshots.sort_values()
    network.snapshot_weightings.sort_index()

    return network

####################################
def manipulate_storage_invest(network, costs=None, wacc=0.05, lifetime=15):
    # default: 4500 € / MW, high 300 €/MW
    crf = (1 / wacc) - (wacc / ((1 + wacc) ** lifetime))
    network.storage_units.capital_cost = costs / crf

def write_lpfile(network=None, path=None):
    network.model.write(path,
                        io_options={'symbolic_solver_labels': True})

def fix_storage_capacity(network, resultspath, n_clusters):  # "network" added
    path = resultspath.strip('daily')
    values = pd.read_csv(path + 'storage_capacity.csv')[n_clusters].values
    network.storage_units.p_nom_max = values
    network.storage_units.p_nom_min = values
    resultspath = 'compare-' + resultspath
<commit_msg>Fix usage of extreme period method<commit_after># -*- coding: utf-8 -*-
# Copyright 2016-2018  Flensburg University of Applied Sciences,
# Europa-Universität Flensburg,
# Centre for Sustainable Energy Systems

# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# File description for read-the-docs
""" This module contains functions for calculating representative days/weeks
based on a pyPSA network object. It is designed to be used for the `lopf`
method. Essentially the tsam package
( https://github.com/FZJ-IEK3-VSA/tsam ), which is developed by
Leander Kotzur is used.

Remaining questions/tasks:

- Does it makes sense to cluster normed values?
- Include scaling method for yearly sums
"""

import pandas as pd
import os
if 'READTHEDOCS' not in os.environ:
    import pyomo.environ as po
    import tsam.timeseriesaggregation as tsam

__copyright__ = ("Flensburg University of Applied Sciences, "
                 "Europa-Universität Flensburg, "
                 "Centre for Sustainable Energy Systems")
__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"
__author__ = "Simon Hilpert"

def snapshot_clustering(network, how='daily', clusters=10):
    """
    """

    network = run(network=network.copy(), n_clusters=clusters,
                  how=how, normed=False)

    return network

def tsam_cluster(timeseries_df,
                 typical_periods=10,
                 how='daily',
                 extremePeriodMethod = 'None'):
    """
    Parameters
    ----------
    df : pd.DataFrame
        DataFrame with timeseries to cluster
    extremePeriodMethod: {'None','append','new_cluster_center',
                           'replace_cluster_center'}, default: 'None'
        Method how to integrate extreme Periods
        into to the typical period profiles.
        None: No integration at all.
        'append': append typical Periods to cluster centers
        'new_cluster_center': add the extreme period as additional cluster
            center. It is checked then for all Periods if they fit better
            to the this new center or their original cluster center.
        'replace_cluster_center': replaces the cluster center of the
            cluster where the extreme period belongs to with the periodly
            profile of the extreme period. (Worst case system design)

    Returns
    -------
    timeseries : pd.DataFrame
        Clustered timeseries
    """

    if how == 'daily':
        hours = 24
        period = ' days'
    if how == 'weekly':
        hours = 168
        period = ' weeks'

    print('Snapshot clustering to ' + str(typical_periods) + period +
          ' using extreme period method: ' + extremePeriodMethod)

    aggregation = tsam.TimeSeriesAggregation(
        timeseries_df,
        noTypicalPeriods=typical_periods,
        extremePeriodMethod = extremePeriodMethod,
        addPeakMin = ['residual_load'],
        addPeakMax = ['residual_load'],
        rescaleClusterPeriods=False,
        hoursPerPeriod=hours,
        clusterMethod='hierarchical')

    timeseries = aggregation.createTypicalPeriods()
    cluster_weights = aggregation.clusterPeriodNoOccur
    clusterOrder =aggregation.clusterOrder
    clusterCenterIndices= aggregation.clusterCenterIndices

    if extremePeriodMethod  == 'new_cluster_center':
        for i in aggregation.extremePeriods.keys():
            clusterCenterIndices.insert(
                    aggregation.extremePeriods[i]['newClusterNo'],
                    aggregation.extremePeriods[i]['stepNo'])

    if extremePeriodMethod  == 'append':
        for i in aggregation.extremePeriods.keys():
            clusterCenterIndices.insert(
                    aggregation.extremePeriods[i]['clusterNo'],
                    aggregation.extremePeriods[i]['stepNo'])

    # get all index for every hour of that day of the clusterCenterIndices
    start = []
    # get the first hour of the clusterCenterIndices (days start with 0)
    for i in clusterCenterIndices:
        start.append(i * hours)

    # get a list with all hours belonging to the clusterCenterIndices
    nrhours = []
    for j in start:
        nrhours.append(j)
        x = 1
        while x < hours:
            j = j + 1
            nrhours.append(j)
            x = x + 1

    # get the origial Datetimeindex
    dates = timeseries_df.iloc[nrhours].index

    #get list of representative days
    representative_day=[]

    #cluster:medoid des jeweiligen Clusters
    dic_clusterCenterIndices = dict(enumerate(clusterCenterIndices))
    for i in clusterOrder:
        representative_day.append(dic_clusterCenterIndices[i])

    #get list of last hour of representative days
    last_hour_datetime=[]
    for i in representative_day:
        last_hour = i * hours + hours - 1
        last_hour_datetime.append(timeseries_df.index[last_hour])

    #create a dataframe (index=nr. of day in a year/candidate)
    df_cluster =  pd.DataFrame({
                        'Cluster': clusterOrder, #Cluster of the day
                        'RepresentativeDay': representative_day, #representative day of the cluster
                        'last_hour_RepresentativeDay': last_hour_datetime}) #last hour of the cluster
    df_cluster.index = df_cluster.index + 1
    df_cluster.index.name = 'Candidate'

    #create a dataframe each timeseries (h) and its candiddate day (i) df_i_h
    nr_day = []
    x = len(timeseries_df.index)/hours+1

    for i in range(1,int(x)):
        j=1
        while j <= hours:
            nr_day.append(i)
            j=j+1
    df_i_h = pd.DataFrame({'Timeseries': timeseries_df.index,
                        'Candidate_day': nr_day})
    df_i_h.set_index('Timeseries',inplace=True)

    return df_cluster, cluster_weights, dates, hours, df_i_h

def run(network, n_clusters=None, how='daily',
        normed=False):
    """
    """

    # calculate clusters
    df_cluster, cluster_weights, dates, hours, df_i_h= tsam_cluster(
                prepare_pypsa_timeseries(network),
                typical_periods=n_clusters,
                how='daily',
                extremePeriodMethod = 'None')
    network.cluster = df_cluster
    network.cluster_ts = df_i_h

    update_data_frames(network, cluster_weights, dates, hours)

    return network

def prepare_pypsa_timeseries(network, normed=False):
    """
    """
    if normed:
        normed_loads = network.loads_t.p_set / network.loads_t.p_set.max()
        normed_loads.columns = 'L' + normed_loads.columns
        normed_renewables = network.generators_t.p_max_pu
        normed_renewables.columns = 'G' + normed_renewables.columns

        df = pd.concat([normed_renewables,
                        normed_loads], axis=1)
    else:
        loads = network.loads_t.p_set.copy()
        loads.columns = 'L' + loads.columns
        renewables = network.generators_t.p_max_pu.mul(
                network.generators.p_nom[
                network.generators_t.p_max_pu.columns], axis = 1).copy()
        renewables.columns = 'G' + renewables.columns
        residual_load=pd.DataFrame()
        residual_load['residual_load']=loads.sum(axis=1)-renewables.sum(axis=1)
        df = pd.concat([renewables, loads, residual_load], axis=1)

    return df

def update_data_frames(network, cluster_weights, dates, hours):
    """ Updates the snapshots, snapshots weights and the dataframes based on
    the original data in the network and the medoids created by clustering
    these original data.

    Parameters
    -----------
    network : pyPSA network object
    cluster_weights: dictionary
    dates: Datetimeindex

    Returns
    -------
    network

    """
    network.snapshot_weightings = network.snapshot_weightings.loc[dates]
    network.snapshots = network.snapshot_weightings.index

    # set new snapshot weights from cluster_weights
    snapshot_weightings = []
    for i in cluster_weights.values():
        x = 0
        while x < hours:
            snapshot_weightings.append(i)
            x += 1
    for i in range(len(network.snapshot_weightings)):
        network.snapshot_weightings[i] = snapshot_weightings[i]

    # put the snapshot in the right order
    network.snapshots.sort_values()
    network.snapshot_weightings.sort_index()

    return network

####################################
def manipulate_storage_invest(network, costs=None, wacc=0.05, lifetime=15):
    # default: 4500 € / MW, high 300 €/MW
    crf = (1 / wacc) - (wacc / ((1 + wacc) ** lifetime))
    network.storage_units.capital_cost = costs / crf

def write_lpfile(network=None, path=None):
    network.model.write(path,
                        io_options={'symbolic_solver_labels': True})

def fix_storage_capacity(network, resultspath, n_clusters):  # "network" added
    path = resultspath.strip('daily')
    values = pd.read_csv(path + 'storage_capacity.csv')[n_clusters].values
    network.storage_units.p_nom_max = values
    network.storage_units.p_nom_min = values
    resultspath = 'compare-' + resultspath
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
CREATED:2014-01-17 16:30:07 by Brian McFee <brm2132@columbia.edu>

Compute segmentation evaluation metrics

Usage:

./segment_eval.py TRUTH.TXT PREDICTION.TXT
'''

import argparse
import sys
import os
from collections import OrderedDict

import mir_eval

def evaluate(truth_file=None, prediction_file=None):
    '''Load data and perform the evaluation'''

    # load the data
    truth_segs, truth_labels    = mir_eval.io.load_annotation(truth_file)
    pred_segs, pred_labels      = mir_eval.io.load_annotation(prediction_file)

    # Convert to boundaries
    annotated_boundaries        = mir_eval.util.segments_to_boundaries(truth_segs)[0]
    predicted_boundaries        = mir_eval.util.segments_to_boundaries(pred_segs)[0]

    # Adjust the predictions to match the annotation time span (0 to end-of-track)

    # The [0] at the end is to just get the boundaries without the labels
    predicted_boundaries        = mir_eval.util.adjust_times(predicted_boundaries,
                                                        t_min=annotated_boundaries.min(),
                                                        t_max=annotated_boundaries.max())[0]

    # Now compute all the metrics

    M = OrderedDict()
    # Boundary detection
    M['P@0.5'], M['R@0.5'], M['F@0.5']  = mir_eval.segment.boundary_detection(annotated_boundaries,
                                                                  predicted_boundaries,
                                                                  window=0.5)

    M['P@3.0'], M['R@3.0'], M['F@3.0']  = mir_eval.segment.boundary_detection(annotated_boundaries,
                                                                 predicted_boundaries,
                                                                 window=3.0)
    # Boundary deviation
    M['True-to-Pred'], M['Pred-to-True'] = mir_eval.segment.boundary_deviation(annotated_boundaries,
                                                                  predicted_boundaries)

    # Pairwise clustering
    M['Pair-P'], M['Pair-R'], M['Pair-F'] = mir_eval.segment.frame_clustering_pairwise(annotated_boundaries,
                                                                         predicted_boundaries)

    # Adjusted rand index
    M['ARI']                     = mir_eval.segment.frame_clustering_ari(annotated_boundaries,
                                                                    predicted_boundaries)

    # Mutual information metrics
    M['MI'], M['AMI'], M['NMI']            = mir_eval.segment.frame_clustering_mi(annotated_boundaries,
                                                                   predicted_boundaries)

    # Conditional entropy metrics
    M['S_Over'], M['S_Under'], M['S_F'] = mir_eval.segment.frame_clustering_nce(annotated_boundaries,
                                                                    predicted_boundaries)

    return M

def print_evaluation(prediction_file, M):
    # And print them
    print os.path.basename(prediction_file)
    for key, value in M.iteritems():
        print '\t%12s:\t%0.3f' % (key, value)

    pass

def process_arguments():
    '''Argparse function to get the program parameters'''

    parser = argparse.ArgumentParser(description='mir_eval segmentation evaluation')

    parser.add_argument(    'truth_file',
                            action      =   'store',
                            help        =   'path to the ground truth annotation')

    parser.add_argument(    'prediction_file',
                            action      =   'store',
                            help        =   'path to the prediction file')

    return vars(parser.parse_args(sys.argv[1:]))

if __name__ == '__main__':
    # Get the parameters
    parameters = process_arguments()

    # Run the beat tracker
    scores = evaluate(**parameters)
    print_evaluation(parameters['prediction_file'], scores)

<commit_msg>removed extra space<commit_after>#!/usr/bin/env python
'''
CREATED:2014-01-17 16:30:07 by Brian McFee <brm2132@columbia.edu>

Compute segmentation evaluation metrics

Usage:

./segment_eval.py TRUTH.TXT PREDICTION.TXT
'''

import argparse
import sys
import os
from collections import OrderedDict

import mir_eval

def evaluate(truth_file=None, prediction_file=None):
    '''Load data and perform the evaluation'''

    # load the data
    truth_segs, truth_labels    = mir_eval.io.load_annotation(truth_file)
    pred_segs, pred_labels      = mir_eval.io.load_annotation(prediction_file)

    # Convert to boundaries
    annotated_boundaries        = mir_eval.util.segments_to_boundaries(truth_segs)[0]
    predicted_boundaries        = mir_eval.util.segments_to_boundaries(pred_segs)[0]

    # Adjust the predictions to match the annotation time span (0 to end-of-track)

    # The [0] at the end is to just get the boundaries without the labels
    predicted_boundaries        = mir_eval.util.adjust_times(predicted_boundaries,
                                                        t_min=annotated_boundaries.min(),
                                                        t_max=annotated_boundaries.max())[0]

    # Now compute all the metrics

    M = OrderedDict()
    # Boundary detection
    M['P@0.5'], M['R@0.5'], M['F@0.5']  = mir_eval.segment.boundary_detection(annotated_boundaries,
                                                                  predicted_boundaries,
                                                                  window=0.5)

    M['P@3.0'], M['R@3.0'], M['F@3.0']  = mir_eval.segment.boundary_detection(annotated_boundaries,
                                                                 predicted_boundaries,
                                                                 window=3.0)
    # Boundary deviation
    M['True-to-Pred'], M['Pred-to-True'] = mir_eval.segment.boundary_deviation(annotated_boundaries,
                                                                  predicted_boundaries)

    # Pairwise clustering
    M['Pair-P'], M['Pair-R'], M['Pair-F'] = mir_eval.segment.frame_clustering_pairwise(annotated_boundaries,
                                                                         predicted_boundaries)

    # Adjusted rand index
    M['ARI']                     = mir_eval.segment.frame_clustering_ari(annotated_boundaries,
                                                                    predicted_boundaries)

    # Mutual information metrics
    M['MI'], M['AMI'], M['NMI']            = mir_eval.segment.frame_clustering_mi(annotated_boundaries,
                                                                   predicted_boundaries)

    # Conditional entropy metrics
    M['S_Over'], M['S_Under'], M['S_F'] = mir_eval.segment.frame_clustering_nce(annotated_boundaries,
                                                                    predicted_boundaries)

    return M

def print_evaluation(prediction_file, M):
    # And print them
    print os.path.basename(prediction_file)
    for key, value in M.iteritems():
        print '\t%12s:\t%0.3f' % (key, value)

    pass

def process_arguments():
    '''Argparse function to get the program parameters'''

    parser = argparse.ArgumentParser(description='mir_eval segmentation evaluation')

    parser.add_argument(    'truth_file',
                            action      =   'store',
                            help        =   'path to the ground truth annotation')

    parser.add_argument(    'prediction_file',
                            action      =   'store',
                            help        =   'path to the prediction file')

    return vars(parser.parse_args(sys.argv[1:]))

if __name__ == '__main__':
    # Get the parameters
    parameters = process_arguments()

    # Run the beat tracker
    scores = evaluate(**parameters)
    print_evaluation(parameters['prediction_file'], scores)

<|endoftext|>


<commit_before>#coding:utf-8

__authors__ = ['"Liu Fei" <fei.liu@cs2c.com.cn>']
__version__ = "V0.1"

'''
# ChangeLog:
#---------------------------------------------------------------------------------
# Version        Date            Desc                            Author
#---------------------------------------------------------------------------------
# V0.1           2014/08/29      初始版本                                                            Liu Fei
#---------------------------------------------------------------------------------
'''

import xmltodict

from BaseAPIs import BaseAPIs
from StorageDomainAPIs import StorageDomainAPIs
from Configs.GlobalConfig import WebBaseApiUrl
from Utils.HttpClient import HttpClient

class DataCenterAPIs(BaseAPIs):
    '''
    @summary: 提供数据中心各种常用操作，通过HttpClient调用相应的REST接口实现。
    '''
    def __init__(self):
        '''
        @summary: 初始化函数，定义datacenter相关API的base_url，如'https://10.1.167.2/api/datacenters'
        '''
        self.base_url = '%s/datacenters' % WebBaseApiUrl

    def searchDataCenterByName(self, dc_name):
        '''
        @summary: 根据数据中心名称查找（调用datacenters接口中的search方法）
        @param dc_name: 数据中心名称
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：搜索到的所有数据中心（dict格式）。
        '''
        return self.searchObject('datacenters', dc_name)

    def getDataCenterIdByName(self, dc_name):
        '''
        @summary: 根据数据中心名称返回其id
        @param dc_name: 数据中心名称
        @return: 数据中心id
        '''
        dc_list = self.searchDataCenterByName(dc_name)
        return dc_list['result']['data_centers']['data_center']['@id']

    def getDataCenterNameById(self, dc_id):
        '''
        @summary: 根据数据中心的id返回其名称
        @param dc_id: 数据中心id
        @return: 数据中心名称
        @todo: 无需实现该功能，后续如果有需要再实现。
        '''
        pass

    def getDataCentersList(self):
        '''
        @summary: 获取全部数据中心列表
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：请求返回的内容（dict格式）。
        '''
        api_url = self.base_url
        method = "GET"
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDataCenterInfo(self, dc_name):
        '''
        @summary: 根据数据中心名称，获取数据中心详细信息
        @param dc_name: 数据中心名称
        @return: 字典：（1）status_code：请求返回码；（2）result：dict形式的数据中心信息
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        api_url = '%s/%s' % (self.base_url, dc_id)
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDataCenterStatus(self, dc_name):
        '''
        @summary: 查看数据中心状态
        @param dc_name: 数据中心名称
        @return: 数据中心状态
        '''
        return self.getDataCenterInfo(dc_name)['result']['data_center']['status']['state']

    def createDataCenter(self, dc_info):
        '''
        @summary: 创建数据中心
        @param dc_info: XML形式的数据中心信息，调用接口时需要传递此xml数据
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：请求返回的内容（dict格式）
        '''
        api_url = self.base_url
        method = 'POST'
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=dc_info)
#         print r.text
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def updateDataCenter(self, dc_name, update_info):
        '''
        @summary: 编辑数据中心
        @param dc_name: 数据中心名称
        @param update_info: 更新的内容
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：请求返回的内容。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        api_url = '%s/%s' % (self.base_url, dc_id)
        method = 'PUT'
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=update_info)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def delDataCenter(self, dc_name, data=None):
        '''
        @summary: 根据数据中心名称删除指定的数据中心（包括普通删除、强制删除等方式，由data中的参数设置确定）
        @param dc_name: 数据中心名称
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：请求返回的内容。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        print dc_id
        api_url = '%s/%s' % (self.base_url, dc_id)
        method = 'DELETE'
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDCStorageDomainsList(self, dc_name):
        '''
        @summary: 获取指定数据中心的存储域列表
        @param dc_name: 数据中心名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        api_url = '%s/%s/storagedomains' % (self.base_url, self.getDataCenterIdByName(dc_name))
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDCStorageDomainInfo(self, dc_name, sd_name):
        '''
        @summary: 获取数据中心的存储域信息
        @param dc_name: 数据中心名称
        @param sd_name: 存储域名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        # 调用StorageDomainAPIs模块中的getStorageDomainIdByName()方法获得存储域id
        sdapi = StorageDomainAPIs()
        sd_id = sdapi.getStorageDomainIdByName(sd_name)
        dc_id = self.getDataCenterIdByName(dc_name)
        api_url = '%s/%s/storagedomains/%s' % (self.base_url, dc_id, sd_id)
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def attachStorageDomainToDC(self, dc_name, sd_name=None, data=None):
        '''
        @summary: 将存储域附加到数据中心
        @param dc_name: 数据中心名称
        @param sd_name: 存储域名称（可选参数，若不指定，则必须按规则在data中指定相应参数）
        @param data: 存储域的xml数据（可选参数，若提供了sd_name，则不需要提供该参数；若未提供sd_name，则需提供相应的xml字符串）
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        api_url = '%s/%s/storagedomains' % (self.base_url, self.getDataCenterIdByName(dc_name))
        method = 'POST'
        if sd_name:
            data = '''<storage_domain><name>%s</name></storage_domain>''' % sd_name
            r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
        elif data:
            r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
        else:
            raise Exception(u'至少需要提供storage_name或xml数据来完成分离操作。')
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def detachStorageDomainFromDC(self, dc_name, sd_name, data=None):
        '''
        @summary: 将维护状态的存储从数据中心分离
        @param dc_name: 数据中心名称
        @param sd_name: 存储名称
        @param data: 可选参数，如<action><async>true</async></action>
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        sdapi = StorageDomainAPIs()
        dc_id = self.getDataCenterIdByName(dc_name)
        sd_id = sdapi.getStorageDomainIdByName(sd_name)
        method = 'DELETE'
        api_url = '%s/%s/storagedomains/%s' % (self.base_url, dc_id, sd_id)
#         data = '''<action><async>true</async></action>'''
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def activeDCStorageDomain(self, dc_name, sd_name):
        '''
        @summary: 激活数据中心中的存储域
        @param dc_name: 数据中心名称
        @param sd_name: 存储域名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        sd_id = StorageDomainAPIs().getStorageDomainIdByName(sd_name)
        api_url = '%s/%s/storagedomains/%s/activate' % (self.base_url, dc_id, sd_id)
        method = 'POST'
        data = '''<action/>'''
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
#         print r.status_code, r.text
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def deactiveDCStorageDomain(self, dc_name, sd_name):
        '''
        @summary: 取消激活数据中心中的存储域
        @param dc_name: 数据中心名称
        @param sd_name: 存储域名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        sd_id = StorageDomainAPIs().getStorageDomainIdByName(sd_name)
        api_url = '%s/%s/storagedomains/%s/deactivate' % (self.base_url, dc_id, sd_id)
        method = 'POST'
        data = '<action/>'
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDCClustersList(self, dc_name):
        '''
        @summary: 获取数据中心的集群列表
        @param dc_name: 数据中心名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容（数据中心的集群列表）。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        api_url = '%s/%s/clusters' % (self.base_url, dc_id)
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDCQuotasList(self, dc_name):
        '''
        @summary: 获取数据中心的Quota列表
        @param dc_name: 数据中心名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容（数据中心的Quota列表）；
        @todo:
        '''
        pass

    def getDCQuotaInfo(self, dc_name, quota_id):
        '''
        @todo:
        '''
        pass

    def createDCQuota(self, dc_name):
        '''
        @summary: 创建数据中心Quota
        @param dc_name: 数据中心名称
        @param quota_name: Quota名称
        @todo:
        '''
        pass

if __name__ == "__main__":
    dcapi = DataCenterAPIs()
    data = '''
    <data_center>
        <name>NewDatacenter</name>
        <storage_type>nfs</storage_type>
        <version minor="1" major="3"/>
    </data_center>
    '''
    dcapi.createDataCenter(data)
#     print dcapi.getDCClustersList('Default')
#     print dcapi.deactiveDCStorageDomain('Default', 'data1')
#     print dcapi.activeDCStorageDomain('Default', 'data1')
#     print dcapi.detachStorageDomainFromDC('Default', 'data1')
#     print dcapi.attachStorageDomainToDC('Default', 'data1')
#     print dcapi.getDataCenterStatus('DC-ISCSI')
#     data = '''<storage_domain><name>data1</name></storage_domain>'''
#     print dcapi.attachStorageDomainToDC('Default', data=data)
#     d = dcapi.getDCStorageDomainInfo('Default', 'data1')
#     print dcapi.getDCStorageDomainsList('Default')
#     print dcapi.getDataCenterIdByName('DC-ISCSI')
#     data = '''
#     <action>
#         <force>true</force>
#         <async>false</async>
#     </action>
#     '''
#     print dcapi.delDataCenter('ABC', data=data)

    <commit_msg>更新DataCenterAPIs.py<commit_after>#coding:utf-8

__authors__ = ['"Liu Fei" <fei.liu@cs2c.com.cn>']
__version__ = "V0.1"

'''
# ChangeLog:
#---------------------------------------------------------------------------------
# Version        Date            Desc                            Author
#---------------------------------------------------------------------------------
# V0.1           2014/08/29      初始版本                                                            Liu Fei
#---------------------------------------------------------------------------------
'''

import xmltodict

from BaseAPIs import BaseAPIs
from StorageDomainAPIs import StorageDomainAPIs
from Configs.GlobalConfig import WebBaseApiUrl
from Utils.HttpClient import HttpClient

class DataCenterAPIs(BaseAPIs):
    '''
    @summary: 提供数据中心各种常用操作，通过HttpClient调用相应的REST接口实现。
    '''
    def __init__(self):
        '''
        @summary: 初始化函数，定义datacenter相关API的base_url，如'https://10.1.167.2/api/datacenters'
        '''
        self.base_url = '%s/datacenters' % WebBaseApiUrl

    def searchDataCenterByName(self, dc_name):
        '''
        @summary: 根据数据中心名称查找（调用datacenters接口中的search方法）
        @param dc_name: 数据中心名称
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：搜索到的所有数据中心（dict格式）。
        '''
        return self.searchObject('datacenters', dc_name)

    def getDataCenterIdByName(self, dc_name):
        '''
        @summary: 根据数据中心名称返回其id
        @param dc_name: 数据中心名称
        @return: 数据中心id
        '''
        dc_list = self.searchDataCenterByName(dc_name)
        return dc_list['result']['data_centers']['data_center']['@id']

    def getDataCenterNameById(self, dc_id):
        '''
        @summary: 根据数据中心的id返回其名称
        @param dc_id: 数据中心id
        @return: 数据中心名称
        '''
        api_url = '%s/%s' % (self.base_url, dc_id)
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        if r.status_code==200:
            return xmltodict.parse(r.text)['data_center']['name']

    def getDataCentersList(self):
        '''
        @summary: 获取全部数据中心列表
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：请求返回的内容（dict格式）。
        '''
        api_url = self.base_url
        method = "GET"
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDataCenterInfo(self, dc_name):
        '''
        @summary: 根据数据中心名称，获取数据中心详细信息
        @param dc_name: 数据中心名称
        @return: 字典：（1）status_code：请求返回码；（2）result：dict形式的数据中心信息
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        api_url = '%s/%s' % (self.base_url, dc_id)
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDataCenterStatus(self, dc_name):
        '''
        @summary: 查看数据中心状态
        @param dc_name: 数据中心名称
        @return: 数据中心状态
        '''
        return self.getDataCenterInfo(dc_name)['result']['data_center']['status']['state']

    def createDataCenter(self, dc_info):
        '''
        @summary: 创建数据中心
        @param dc_info: XML形式的数据中心信息，调用接口时需要传递此xml数据
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：请求返回的内容（dict格式）
        '''
        api_url = self.base_url
        method = 'POST'
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=dc_info)
#         print r.text
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def updateDataCenter(self, dc_name, update_info):
        '''
        @summary: 编辑数据中心
        @param dc_name: 数据中心名称
        @param update_info: 更新的内容
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：请求返回的内容。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        api_url = '%s/%s' % (self.base_url, dc_id)
        method = 'PUT'
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=update_info)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def delDataCenter(self, dc_name, data=None):
        '''
        @summary: 根据数据中心名称删除指定的数据中心（包括普通删除、强制删除等方式，由data中的参数设置确定）
        @param dc_name: 数据中心名称
        @return: 字典，包括：（1）status_code：http请求返回码；（2）result：请求返回的内容。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        print dc_id
        api_url = '%s/%s' % (self.base_url, dc_id)
        method = 'DELETE'
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDCStorageDomainsList(self, dc_name):
        '''
        @summary: 获取指定数据中心的存储域列表
        @param dc_name: 数据中心名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        api_url = '%s/%s/storagedomains' % (self.base_url, self.getDataCenterIdByName(dc_name))
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDCStorageDomainInfo(self, dc_name, sd_name):
        '''
        @summary: 获取数据中心的存储域信息
        @param dc_name: 数据中心名称
        @param sd_name: 存储域名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        # 调用StorageDomainAPIs模块中的getStorageDomainIdByName()方法获得存储域id
        sdapi = StorageDomainAPIs()
        sd_id = sdapi.getStorageDomainIdByName(sd_name)
        dc_id = self.getDataCenterIdByName(dc_name)
        api_url = '%s/%s/storagedomains/%s' % (self.base_url, dc_id, sd_id)
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def attachStorageDomainToDC(self, dc_name, sd_name=None, data=None):
        '''
        @summary: 将存储域附加到数据中心
        @param dc_name: 数据中心名称
        @param sd_name: 存储域名称（可选参数，若不指定，则必须按规则在data中指定相应参数）
        @param data: 存储域的xml数据（可选参数，若提供了sd_name，则不需要提供该参数；若未提供sd_name，则需提供相应的xml字符串）
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        api_url = '%s/%s/storagedomains' % (self.base_url, self.getDataCenterIdByName(dc_name))
        method = 'POST'
        if sd_name:
            data = '''<storage_domain><name>%s</name></storage_domain>''' % sd_name
            r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
        elif data:
            r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
        else:
            raise Exception(u'至少需要提供storage_name或xml数据来完成分离操作。')
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def detachStorageDomainFromDC(self, dc_name, sd_name, data=None):
        '''
        @summary: 将维护状态的存储从数据中心分离
        @param dc_name: 数据中心名称
        @param sd_name: 存储名称
        @param data: 可选参数，如<action><async>true</async></action>
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        sdapi = StorageDomainAPIs()
        dc_id = self.getDataCenterIdByName(dc_name)
        sd_id = sdapi.getStorageDomainIdByName(sd_name)
        method = 'DELETE'
        api_url = '%s/%s/storagedomains/%s' % (self.base_url, dc_id, sd_id)
#         data = '''<action><async>true</async></action>'''
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def activeDCStorageDomain(self, dc_name, sd_name):
        '''
        @summary: 激活数据中心中的存储域
        @param dc_name: 数据中心名称
        @param sd_name: 存储域名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        sd_id = StorageDomainAPIs().getStorageDomainIdByName(sd_name)
        api_url = '%s/%s/storagedomains/%s/activate' % (self.base_url, dc_id, sd_id)
        method = 'POST'
        data = '''<action/>'''
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
#         print r.status_code, r.text
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def deactiveDCStorageDomain(self, dc_name, sd_name):
        '''
        @summary: 取消激活数据中心中的存储域
        @param dc_name: 数据中心名称
        @param sd_name: 存储域名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        sd_id = StorageDomainAPIs().getStorageDomainIdByName(sd_name)
        api_url = '%s/%s/storagedomains/%s/deactivate' % (self.base_url, dc_id, sd_id)
        method = 'POST'
        data = '<action/>'
        r = HttpClient.sendRequest(method=method, api_url=api_url, data=data)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDCClustersList(self, dc_name):
        '''
        @summary: 获取数据中心的集群列表
        @param dc_name: 数据中心名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容（数据中心的集群列表）。
        '''
        dc_id = self.getDataCenterIdByName(dc_name)
        api_url = '%s/%s/clusters' % (self.base_url, dc_id)
        method = 'GET'
        r = HttpClient.sendRequest(method=method, api_url=api_url)
        return {'status_code':r.status_code, 'result':xmltodict.parse(r.text)}

    def getDCQuotasList(self, dc_name):
        '''
        @summary: 获取数据中心的Quota列表
        @param dc_name: 数据中心名称
        @return: 字典，（1）status_code：请求返回状态码；（2）result：请求返回的内容（数据中心的Quota列表）；
        @todo:
        '''
        pass

    def getDCQuotaInfo(self, dc_name, quota_id):
        '''
        @todo:
        '''
        pass

    def createDCQuota(self, dc_name):
        '''
        @summary: 创建数据中心Quota
        @param dc_name: 数据中心名称
        @param quota_name: Quota名称
        @todo:
        '''
        pass

if __name__ == "__main__":
    dcapi = DataCenterAPIs()
    data = '''
    <data_center>
        <name>NewDatacenter</name>
        <storage_type>nfs</storage_type>
        <version minor="1" major="3"/>
    </data_center>
    '''
    dcapi.createDataCenter(data)
    print dcapi.getDataCenterNameById('5849b030-626e-47cb-ad90-3ce782d831b3')
#     print dcapi.getDCClustersList('Default')
#     print dcapi.deactiveDCStorageDomain('Default', 'data1')
#     print dcapi.activeDCStorageDomain('Default', 'data1')
#     print dcapi.detachStorageDomainFromDC('Default', 'data1')
#     print dcapi.attachStorageDomainToDC('Default', 'data1')
#     print dcapi.getDataCenterStatus('DC-ISCSI')
#     data = '''<storage_domain><name>data1</name></storage_domain>'''
#     print dcapi.attachStorageDomainToDC('Default', data=data)
#     d = dcapi.getDCStorageDomainInfo('Default', 'data1')
#     print dcapi.getDCStorageDomainsList('Default')
#     print dcapi.getDataCenterIdByName('DC-ISCSI')
#     data = '''
#     <action>
#         <force>true</force>
#         <async>false</async>
#     </action>
#     '''
#     print dcapi.delDataCenter('ABC', data=data)

    <|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
:py:mod:`pbs.py` - PBS cluster routines
---------------------------------------

'''

from __future__ import division, print_function, absolute_import, unicode_literals
from .aux import *
from .k2 import GetData
from ...config import EVEREST_SRC, EVEREST_DAT, EVEREST_DEV
from ...utils import ExceptionHook, FunctionWrapper
from ...pool import Pool
import os, sys, subprocess
import pickle
import logging
log = logging.getLogger(__name__)

# Constants
RED =    '\033[0;31m'
GREEN =  '\033[0;32m'
BLACK =  '\033[0m'
BLUE =   '\033[0;34m'

def Download(campaign = 0, queue = 'build', email = None, walltime = 8, **kwargs):
  '''
  Submits a cluster job to the build queue to download all TPFs for a given
  campaign.

  :param int campaign: The `K2` campaign to run
  :param str queue: The name of the queue to submit to. Default `build`
  :param str email: The email to send job status notifications to. Default `None`
  :param int walltime: The number of hours to request. Default `8`

  '''

  # Figure out the subcampaign
  if type(campaign) is int:
    subcampaign = -1
  elif type(campaign) is float:
    x, y = divmod(campaign, 1)
    campaign = int(x)
    subcampaign = round(y * 10)
  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_SRC, 'missions', 'k2', 'download.pbs')
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_DAT=%s,CAMPAIGN=%d,SUBCAMPAIGN=%d' % (EVEREST_DAT, campaign, subcampaign)
  if subcampaign == -1:
    str_name = 'download_c%02d' % campaign
  else:
    str_name = 'download_c%02d.%d' % (campaign, subcampaign)
  str_out = os.path.join(EVEREST_DAT, 'k2', str_name + '.log')
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', str_name,
               '-l', str_w]
  if email is not None: qsub_args.append(['-M', email, '-m', 'ae'])
  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def _Download(campaign, subcampaign):
  '''
  Download all stars from a given campaign. This is
  called from ``missions/k2/download.pbs``

  '''

  # Are we doing a subcampaign?
  if subcampaign != -1:
    campaign = campaign + 0.1 * subcampaign
  # Get all star IDs for this campaign
  stars = [s[0] for s in GetK2Campaign(campaign)]
  nstars = len(stars)
  # Download the TPF data for each one
  for i, EPIC in enumerate(stars):
    print("Downloading data for EPIC %d (%d/%d)..." % (EPIC, i + 1, nstars))
    if not os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % int(campaign),
                         ('%09d' % EPIC)[:4] + '00000', ('%09d' % EPIC)[4:],
                         'data.npz')):
      try:
        GetData(EPIC, download_only = True)
      except KeyboardInterrupt:
        sys.exit()
      except:
        # Some targets could be corrupted...
        print("ERROR downloading EPIC %d." % EPIC)
        continue

def Run(campaign = 0, nodes = 5, ppn = 12, walltime = 100,
        mpn = None, email = None, queue = None, **kwargs):
  '''
  Submits a cluster job to compute and plot data for all targets in a given campaign.

  :param campaign: The K2 campaign number. If this is an :py:class:`int`, returns \
                   all targets in that campaign. If a :py:class:`float` in the form \
                   `X.Y`, runs the `Y^th` decile of campaign `X`.
  :param str queue: The queue to submit to. Default `None` (default queue)
  :param str email: The email to send job status notifications to. Default `None`
  :param int walltime: The number of hours to request. Default `100`
  :param int nodes: The number of nodes to request. Default `5`
  :param int ppn: The number of processors per node to request. Default `12`
  :param int mpn: Memory per node in gb to request. Default no setting.

  '''

  # Figure out the subcampaign
  if type(campaign) is int:
    subcampaign = -1
  elif type(campaign) is float:
    x, y = divmod(campaign, 1)
    campaign = int(x)
    subcampaign = round(y * 10)

  # DEV hack: limit backfill jobs to 4 hours
  if EVEREST_DEV and (queue == 'bf'):
    walltime = min(4, walltime)

  # Convert kwargs to string
  strkwargs = pickle.dumps(kwargs, 0).decode('utf-8')

  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_SRC, 'missions', 'k2', 'run.pbs')
  if mpn is not None:
    str_n = 'nodes=%d:ppn=%d,feature=%dcore,mem=%dgb' % (nodes, ppn, ppn, mpn * nodes)
  else:
    str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)
  str_w = 'walltime=%d:00:00' % walltime
  str_v = "EVEREST_DAT=%s,NODES=%d,CAMPAIGN=%d,SUBCAMPAIGN=%d,STRKWARGS='%s'" % (EVEREST_DAT,
          nodes, campaign, subcampaign, strkwargs)
  if subcampaign == -1:
    str_name = 'c%02d' % campaign
  else:
    str_name = 'c%02d.%d' % (campaign, subcampaign)
  str_out = os.path.join(EVEREST_DAT, 'k2', str_name + '.log')
  qsub_args = ['qsub', pbsfile,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', str_name,
               '-l', str_n,
               '-l', str_w]
  if email is not None:
    qsub_args.append(['-M', email, '-m', 'ae'])
  if queue is not None:
    qsub_args += ['-q', queue]

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def _Run(campaign, subcampaign, strkwargs):
  '''
  The actual function that runs a given campaign; this must
  be called from ``missions/k2/run.pbs``.

  '''

  # Get kwargs from string
  kwargs = pickle.loads(strkwargs.encode('utf-8'))

  # Model wrapper
  m = FunctionWrapper(EverestModel, **kwargs)

  # Set up our custom exception handler
  sys.excepthook = ExceptionHook
  # Initialize our multiprocessing pool
  with Pool() as pool:
    # Are we doing a subcampaign?
    if subcampaign != -1:
      campaign = campaign + 0.1 * subcampaign
    # Get all the stars
    stars = GetK2Campaign(campaign, epics_only = True)
    # Run
    pool.map(m, stars)

def Status(campaign = range(18), model = 'nPLD', purge = False, **kwargs):
  '''
  Shows the progress of the de-trending runs for the specified campaign(s).

  '''

  if not hasattr(campaign, '__len__'):
    if type(campaign) is int:
      # Return the subcampaigns
      all_stars = [s for s in GetK2Campaign(campaign, split = True, epics_only = True)]
      campaign = [campaign + 0.1 * n for n in range(10)]
    else:
      all_stars = [[s for s in GetK2Campaign(campaign, epics_only = True)]]
      campaign = [campaign]
  else:
    all_stars = [[s for s in GetK2Campaign(c, epics_only = True)] for c in campaign]
  print("CAMP      TOTAL      DOWNLOADED    PROCESSED    ERRORS")
  print("----      -----      ----------    ---------    ------")
  for c, stars in zip(campaign, all_stars):
    if len(stars) == 0:
      continue
    down = 0
    proc = 0
    err = 0
    bad = []
    remain = []
    total = len(stars)
    if os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c)):
      path = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c)
      for folder in os.listdir(path):
        for subfolder in os.listdir(os.path.join(path, folder)):
          if os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c, folder, subfolder, 'data.npz')):
            if int(folder[:4] + subfolder) in stars:
              down += 1
              if os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c, folder, subfolder, model + '.npz')):
                proc += 1
              elif os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c, folder, subfolder, model + '.err')):
                err += 1
                bad.append(folder[:4] + subfolder)
                if purge:
                  os.remove(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c, folder, subfolder, model + '.err'))
              else:
                remain.append(folder[:4] + subfolder)
    if proc == total:
      cc = ct = cd = cp = ce = GREEN
    else:
      cc = BLACK
      ct = BLACK
      cd = BLACK if down < total else BLUE
      cp = BLACK if proc < down or proc == 0 else BLUE
      ce = RED if err > 0 else BLACK
    if type(c) is int:
      print("%s{:>4d}   \033[0m%s{:>8d}\033[0m%s{:>16d}\033[0m%s{:>13d}\033[0m%s{:>10d}\033[0m".format(c, total, down, proc, err)
            % (cc, ct, cd, cp, ce))
    else:
      print("%s{:>4.1f}   \033[0m%s{:>8d}\033[0m%s{:>16d}\033[0m%s{:>13d}\033[0m%s{:>10d}\033[0m".format(c, total, down, proc, err)
            % (cc, ct, cd, cp, ce))
    if len(remain) <= 25 and len(remain) > 0 and len(campaign) == 1:
      remain.extend(["         "] * (4 - (len(remain) % 4)))
      print()
      for A, B, C, D in zip(remain[::4],remain[1::4],remain[2::4],remain[3::4]):
        if A == remain[0]:
          print("REMAIN:  %s   %s   %s   %s" % (A, B, C, D))
          print()
        else:
          print("         %s   %s   %s   %s" % (A, B, C, D))
          print()
    if len(bad) and len(campaign) == 1:
      bad.extend(["         "] * (4 - (len(bad) % 4)))
      print()
      for A, B, C, D in zip(bad[::4],bad[1::4],bad[2::4],bad[3::4]):
        if A == bad[0]:
          print("ERRORS:  %s   %s   %s   %s" % (A, B, C, D))
          print()
        else:
          print("         %s   %s   %s   %s" % (A, B, C, D))
          print()

def EverestModel(ID, model = 'nPLD', **kwargs):
  '''

  '''

  from ... import model as everest_model
  getattr(everest_model, model)(ID, **kwargs)
  return True<commit_msg>kwargs bytes bugfix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
:py:mod:`pbs.py` - PBS cluster routines
---------------------------------------

'''

from __future__ import division, print_function, absolute_import, unicode_literals
from .aux import *
from .k2 import GetData
from ...config import EVEREST_SRC, EVEREST_DAT, EVEREST_DEV
from ...utils import ExceptionHook, FunctionWrapper
from ...pool import Pool
import os, sys, subprocess
import pickle
import logging
log = logging.getLogger(__name__)

# Constants
RED =    '\033[0;31m'
GREEN =  '\033[0;32m'
BLACK =  '\033[0m'
BLUE =   '\033[0;34m'

def Download(campaign = 0, queue = 'build', email = None, walltime = 8, **kwargs):
  '''
  Submits a cluster job to the build queue to download all TPFs for a given
  campaign.

  :param int campaign: The `K2` campaign to run
  :param str queue: The name of the queue to submit to. Default `build`
  :param str email: The email to send job status notifications to. Default `None`
  :param int walltime: The number of hours to request. Default `8`

  '''

  # Figure out the subcampaign
  if type(campaign) is int:
    subcampaign = -1
  elif type(campaign) is float:
    x, y = divmod(campaign, 1)
    campaign = int(x)
    subcampaign = round(y * 10)
  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_SRC, 'missions', 'k2', 'download.pbs')
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_DAT=%s,CAMPAIGN=%d,SUBCAMPAIGN=%d' % (EVEREST_DAT, campaign, subcampaign)
  if subcampaign == -1:
    str_name = 'download_c%02d' % campaign
  else:
    str_name = 'download_c%02d.%d' % (campaign, subcampaign)
  str_out = os.path.join(EVEREST_DAT, 'k2', str_name + '.log')
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', str_name,
               '-l', str_w]
  if email is not None: qsub_args.append(['-M', email, '-m', 'ae'])
  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def _Download(campaign, subcampaign):
  '''
  Download all stars from a given campaign. This is
  called from ``missions/k2/download.pbs``

  '''

  # Are we doing a subcampaign?
  if subcampaign != -1:
    campaign = campaign + 0.1 * subcampaign
  # Get all star IDs for this campaign
  stars = [s[0] for s in GetK2Campaign(campaign)]
  nstars = len(stars)
  # Download the TPF data for each one
  for i, EPIC in enumerate(stars):
    print("Downloading data for EPIC %d (%d/%d)..." % (EPIC, i + 1, nstars))
    if not os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % int(campaign),
                         ('%09d' % EPIC)[:4] + '00000', ('%09d' % EPIC)[4:],
                         'data.npz')):
      try:
        GetData(EPIC, download_only = True)
      except KeyboardInterrupt:
        sys.exit()
      except:
        # Some targets could be corrupted...
        print("ERROR downloading EPIC %d." % EPIC)
        continue

def Run(campaign = 0, nodes = 5, ppn = 12, walltime = 100,
        mpn = None, email = None, queue = None, **kwargs):
  '''
  Submits a cluster job to compute and plot data for all targets in a given campaign.

  :param campaign: The K2 campaign number. If this is an :py:class:`int`, returns \
                   all targets in that campaign. If a :py:class:`float` in the form \
                   `X.Y`, runs the `Y^th` decile of campaign `X`.
  :param str queue: The queue to submit to. Default `None` (default queue)
  :param str email: The email to send job status notifications to. Default `None`
  :param int walltime: The number of hours to request. Default `100`
  :param int nodes: The number of nodes to request. Default `5`
  :param int ppn: The number of processors per node to request. Default `12`
  :param int mpn: Memory per node in gb to request. Default no setting.

  '''

  # Figure out the subcampaign
  if type(campaign) is int:
    subcampaign = -1
  elif type(campaign) is float:
    x, y = divmod(campaign, 1)
    campaign = int(x)
    subcampaign = round(y * 10)

  # DEV hack: limit backfill jobs to 4 hours
  if EVEREST_DEV and (queue == 'bf'):
    walltime = min(4, walltime)

  # Convert kwargs to string
  strkwargs = pickle.dumps(kwargs, 0).decode('utf-8').replace('\n', '\\n')

  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_SRC, 'missions', 'k2', 'run.pbs')
  if mpn is not None:
    str_n = 'nodes=%d:ppn=%d,feature=%dcore,mem=%dgb' % (nodes, ppn, ppn, mpn * nodes)
  else:
    str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)
  str_w = 'walltime=%d:00:00' % walltime
  str_v = "EVEREST_DAT=%s,NODES=%d,CAMPAIGN=%d,SUBCAMPAIGN=%d,STRKWARGS='%s'" % (EVEREST_DAT,
          nodes, campaign, subcampaign, strkwargs)
  if subcampaign == -1:
    str_name = 'c%02d' % campaign
  else:
    str_name = 'c%02d.%d' % (campaign, subcampaign)
  str_out = os.path.join(EVEREST_DAT, 'k2', str_name + '.log')
  qsub_args = ['qsub', pbsfile,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', str_name,
               '-l', str_n,
               '-l', str_w]
  if email is not None:
    qsub_args.append(['-M', email, '-m', 'ae'])
  if queue is not None:
    qsub_args += ['-q', queue]

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def _Run(campaign, subcampaign, strkwargs):
  '''
  The actual function that runs a given campaign; this must
  be called from ``missions/k2/run.pbs``.

  '''

  # Get kwargs from string
  kwargs = pickle.loads(strkwargs.replace('\\n', '\n').encode('utf-8'))

  # Model wrapper
  m = FunctionWrapper(EverestModel, **kwargs)

  # Set up our custom exception handler
  sys.excepthook = ExceptionHook
  # Initialize our multiprocessing pool
  with Pool() as pool:
    # Are we doing a subcampaign?
    if subcampaign != -1:
      campaign = campaign + 0.1 * subcampaign
    # Get all the stars
    stars = GetK2Campaign(campaign, epics_only = True)
    # Run
    pool.map(m, stars)

def Status(campaign = range(18), model = 'nPLD', purge = False, **kwargs):
  '''
  Shows the progress of the de-trending runs for the specified campaign(s).

  '''

  if not hasattr(campaign, '__len__'):
    if type(campaign) is int:
      # Return the subcampaigns
      all_stars = [s for s in GetK2Campaign(campaign, split = True, epics_only = True)]
      campaign = [campaign + 0.1 * n for n in range(10)]
    else:
      all_stars = [[s for s in GetK2Campaign(campaign, epics_only = True)]]
      campaign = [campaign]
  else:
    all_stars = [[s for s in GetK2Campaign(c, epics_only = True)] for c in campaign]
  print("CAMP      TOTAL      DOWNLOADED    PROCESSED    ERRORS")
  print("----      -----      ----------    ---------    ------")
  for c, stars in zip(campaign, all_stars):
    if len(stars) == 0:
      continue
    down = 0
    proc = 0
    err = 0
    bad = []
    remain = []
    total = len(stars)
    if os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c)):
      path = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c)
      for folder in os.listdir(path):
        for subfolder in os.listdir(os.path.join(path, folder)):
          if os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c, folder, subfolder, 'data.npz')):
            if int(folder[:4] + subfolder) in stars:
              down += 1
              if os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c, folder, subfolder, model + '.npz')):
                proc += 1
              elif os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c, folder, subfolder, model + '.err')):
                err += 1
                bad.append(folder[:4] + subfolder)
                if purge:
                  os.remove(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c, folder, subfolder, model + '.err'))
              else:
                remain.append(folder[:4] + subfolder)
    if proc == total:
      cc = ct = cd = cp = ce = GREEN
    else:
      cc = BLACK
      ct = BLACK
      cd = BLACK if down < total else BLUE
      cp = BLACK if proc < down or proc == 0 else BLUE
      ce = RED if err > 0 else BLACK
    if type(c) is int:
      print("%s{:>4d}   \033[0m%s{:>8d}\033[0m%s{:>16d}\033[0m%s{:>13d}\033[0m%s{:>10d}\033[0m".format(c, total, down, proc, err)
            % (cc, ct, cd, cp, ce))
    else:
      print("%s{:>4.1f}   \033[0m%s{:>8d}\033[0m%s{:>16d}\033[0m%s{:>13d}\033[0m%s{:>10d}\033[0m".format(c, total, down, proc, err)
            % (cc, ct, cd, cp, ce))
    if len(remain) <= 25 and len(remain) > 0 and len(campaign) == 1:
      remain.extend(["         "] * (4 - (len(remain) % 4)))
      print()
      for A, B, C, D in zip(remain[::4],remain[1::4],remain[2::4],remain[3::4]):
        if A == remain[0]:
          print("REMAIN:  %s   %s   %s   %s" % (A, B, C, D))
          print()
        else:
          print("         %s   %s   %s   %s" % (A, B, C, D))
          print()
    if len(bad) and len(campaign) == 1:
      bad.extend(["         "] * (4 - (len(bad) % 4)))
      print()
      for A, B, C, D in zip(bad[::4],bad[1::4],bad[2::4],bad[3::4]):
        if A == bad[0]:
          print("ERRORS:  %s   %s   %s   %s" % (A, B, C, D))
          print()
        else:
          print("         %s   %s   %s   %s" % (A, B, C, D))
          print()

def EverestModel(ID, model = 'nPLD', **kwargs):
  '''

  '''

  from ... import model as everest_model
  getattr(everest_model, model)(ID, **kwargs)
  return True<|endoftext|>