<commit_before>#!/usr/bin/env python

'''
Created on 12 Mar 2013

@author: nathan
'''
import roslib
roslib.load_manifest('sf_controller')

import time
import math
from threading import Thread

import rospy
import actionlib
import sf_controller_msgs.msg
import sf_lights_msgs.msg
from dynamixel_msgs.msg import JointState
from geometry_msgs.msg import PoseStamped, Twist
from p2os_driver.msg import MotorState
from tf.transformations import quaternion_from_euler
from std_msgs.msg import Float64
from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal

class SunflowerAction(object):

    _actionHandles = {}

    def __init__(self, name):
        self._action_name = name
        self._as = actionlib.SimpleActionServer(self._action_name, sf_controller_msgs.msg.SunflowerAction, execute_cb=self.execute_cb, auto_start=False)
        self._as.start()
        rospy.loginfo("Started Sunflower Controller ActionServer")
        self._pubs = self._connectToJoints()
        self._subs = {}
        (self._cmdVel, self._motorState) = self._connectToWheels()
        self._feedback = sf_controller_msgs.msg.SunflowerFeedback()
        self._result = sf_controller_msgs.msg.SunflowerResult()

    def __del__(self):
        for pub in self._pubs:
            pub.unregister()

    def park(self):
        g = sf_controller_msgs.msg.SunflowerAction()
        g.component = 'head'
        g.positions = [0, 0, -1.67, 1.67]
        self.execute_cb(g)
        g = sf_controller_msgs.msg.SunflowerAction()
        g.component = 'tray'
        g.positions = [0, ]
        self.execute_cb(g)
        g = sf_controller_msgs.msg.SunflowerAction()
        g.component = 'lights'
        g.positions = [0, 0, 0]
        self.execute_cb(g)

    def _connectToWheels(self):
        return (rospy.Publisher('cmd_vel', Twist), rospy.Publisher('cmd_motor_state', MotorState))

    def _connectToJoints(self):
        pubs = {}
        joints = rospy.get_param('/sf_controller', None)
        if joints:
            for value in joints.values():
                if 'joint_names' in value:
                    for jointName in value['joint_names']:
                        topic = jointName + '_controller'
                        if topic not in pubs:
                            pubs[topic] = rospy.Publisher(topic + '/command', Float64)
        else:
            rospy.logerr("%s:  No joints received from parameter server",
                          self._action_name)

        return pubs

    def execute_cb(self, goal):
        if goal.component == 'light':
            result = self.setlight(goal.jointPositions)
        elif goal.action == 'move':
            result = self.move(goal)
        elif goal.action == 'init':
            result = self.init(goal.component)
        elif goal.action == 'stop':
            self.stop(goal.component)
            result = True
        elif goal.action == 'park':
            self.park()
            result = True
        else:
            rospy.logwarn("Unknown action %s", goal.action)
            self._result.result = -1
            self._as.set_aborted(self._result)

        if result:
            self._result.result = 0
            self._as.set_succeeded(self._result)
        else:
            self._result.result = -1
            self._as.set_aborted(self._result)

    def init(self, name):
        if name == 'base' or name == 'base_direct':
            self._motorState.publish(MotorState(1))

        self._as.set_succeeded(self._result)

    def stop(self, name):
        rospy.loginfo("%s: Stopping %s",
            self._action_name,
            name)
        if name == 'base':
            client = actionlib.SimpleActionClient('/lights', sf_lights_msgs.msg.LightsAction)
            client.wait_for_server()
            client.cancel_all_goals()
        elif name == 'light':
            client = actionlib.SimpleActionClient('/move_base', MoveBaseAction)
            client.wait_for_server()
            client.cancel_all_goals()
        else:
            pass

        self._as.set_succeeded(self._result)

    def setlight(self, color):
        client = actionlib.SimpleActionClient('/lights', sf_lights_msgs.msg.LightsAction)
        client.wait_for_server()
        goal = sf_lights_msgs.msg.LightsGoal(rgb=color)
        handle = _ActionHandle(client)
        client.send_goal(goal)
        handle.wait()
        handle.result

    def move(self, goal):
        joints = goal.jointPositions

        if(goal.namedPosition != '' and goal.namedPosition != None):
            param = '/sf_controller/' + goal.component + '/' + goal.namedPosition
            if(rospy.has_param(param)):
                joints = rospy.get_param(param)[0]

        rospy.loginfo("%s: Setting %s to %s",
                self._action_name,
                goal.component,
                goal.namedPosition or joints)

        if goal.component == 'base':
            result = self.navigate(goal, joints)
        elif goal.component == 'base_direct':
            result = self.moveBase(goal, joints)
        else:
            result = self.moveJoints(goal, joints)

        rospy.logdebug("%s: '%s to %s' Result:%s",
                self._action_name,
                goal.component,
                goal.namedPosition or joints,
                result)

        return result == 3

    def moveBase(self, goal, positions):
        # TODO: These should be in a config file
        maxTrans = 1.5
        LINEAR_RATE = 0.3  # [m/s]
        maxRot = 2 * math.pi
        ROTATION_RATE = 0.5  # [rad/s]

        rotation = positions[0]
        linear = positions[1]

        rospy.loginfo("Moving base %s rad and %s meters", rotation, linear)

        # step 0: check validity of parameters:
        if not isinstance(rotation, (int, float)) or not isinstance(linear, (int, float)):
                rospy.logerr("Non-numeric rotation list, aborting moveBase")
                return False
        if abs(linear) >= maxTrans:
                rospy.logerr("Maximal relative translation step exceeded(max: %sm), aborting moveBase" % maxTrans)
                return False
        if abs(rotation) >= maxRot:
                rospy.logerr("Maximal relative rotation step exceeded(max: %srad), aborting moveBase" % maxRot)
                return False

        # step 1: determine duration of motion so that upper thresholds for both translational as well as rotational velocity are not exceeded
        duration_trans_sec = abs(linear) / LINEAR_RATE
        duration_rot_sec = abs(rotation) / ROTATION_RATE
        duration_sec = max(duration_trans_sec, duration_rot_sec)

        # step 2: determine actual velocities based on calculated duration
        x_vel = linear / duration_sec
        rot_vel = rotation / duration_sec

        # step 3: send constant velocity command to base_controller for the calculated duration of motion
        # pub = rospy.Publisher('cmd_vel', Twist)
        twist = Twist()
        twist.linear.x = x_vel
        twist.angular.z = rot_vel
        duration_ros = rospy.Duration.from_sec(duration_sec)  # duration of motion in ROS time

        r = rospy.Rate(50)
        end_time = rospy.Time.now() + duration_ros
        self._cmdVel.publish(twist)
        while not rospy.is_shutdown() and rospy.Time.now() < end_time:
            # pub.publish(twist) #p2os has issues if you republish, seems to continue using last received cmd_vel
            if self._as.isPreemptRequested():
                rospy.loginfo('%s: Preempted' % self._action_name)
                self._as.set_preempted()
                return 2
            r.sleep()

        self._cmdVel.publish(Twist())  # send a stop command, see above comment

        return 3

    def navigate(self, goal, positions):
        pose = PoseStamped()
        pose.header.stamp = rospy.Time.now()
        pose.header.frame_id = "/map"
        pose.pose.position.x = positions[0]
        pose.pose.position.y = positions[1]
        pose.pose.position.z = 0.0
        q = quaternion_from_euler(0, 0, positions[2])
        pose.pose.orientation.x = q[0]
        pose.pose.orientation.y = q[1]
        pose.pose.orientation.z = q[2]
        pose.pose.orientation.w = q[3]

        client = actionlib.SimpleActionClient('/move_base', MoveBaseAction)
        client_goal = MoveBaseGoal()
        client_goal.target_pose = pose

        client.wait_for_server()
        rospy.loginfo("%s: Navigating to %s",
                self._action_name,
                pose)

        handle = _ActionHandle(client)
        client.send_goal(client_goal)
        handle.wait()
        return handle.result

    def moveJoints(self, goal, positions):
        subs = []
        pubs = {}

        component_name = '/sf_controller/' + goal.component
        try:
            joint_names = rospy.get_param(component_name + '/joint_names')
        except KeyError:
            # assume component is a named joint
            joint_names = [goal.component, ]
        for i in range(0, len(joint_names)):
            topic = joint_names[i] + '_controller'
            if topic not in self._pubs:
                rospy.logerr('Undefined joint controller %s', topic)
                return False

            subs.append(RosSubscriber(topic + '/state', JointState))
            pubs[topic] = self._pubs[topic]
            self._pubs[topic].publish(Float64(positions[i]))

        handle = _SubscriberHandle(subs, pubs)
        handle.wait()
        return handle.result

#------------------- action_handle section -------------------#
# Subscriber handle class.
#
# The subscriber handle is used to implement asynchronous behaviour within the script.
class _SubscriberHandle(object):
        # Initialises the action handle.
        def __init__(self, subscribers, publishers):
            self._subscribers = subscribers
            self._publishers = publishers
            self._waiting = False
            self._result = None

        @property
        def result(self):
            if self._waiting:
                return None

            return self._result

        def wait(self, duration=None):
            t = self.waitAsync(duration)
            t.join()

        def waitAsync(self, duration=None):
            thread = Thread(target=self._wait_for_finished, args=(duration,))
            thread.setDaemon(True)
            thread.start()
            return thread

        def _wait_for_finished(self, duration):
            self._waiting = True
            reached = True
            while True:
                r = rospy.Rate(100)
                for sub in self._subscribers:
                    if rospy.is_shutdown():
                        return False

                    while not sub.hasNewMessage:
                        r.sleep()

                    if sub.lastMessage.is_moving:
                        continue

                    reached &= abs(sub.lastMessage.error) < 0.03
                    if not reached:
                        rospy.logwarn("%s unable to reach %s.  Error value: %s" % (sub.topic, sub.lastMessage.goal_pos, sub.lastMessage.error))

                    self._subscribers.remove(sub)

                if not self._subscribers:
                    break

            self._result = 3 if reached else 4

        def cancel(self):
            for sub in self._subscribers:
                self._publishers[sub.topic].publish(Float64(sub.lastMessage.current_pos))

#------------------- action_handle section -------------------#
# # Action handle class.
#
# The action handle is used to implement asynchronous behaviour within the script.
class _ActionHandle(object):
        # Initialises the action handle.
        def __init__(self, simpleActionClient):
            self._client = simpleActionClient
            self._waiting = False
            self._result = None

        @property
        def result(self):
            if self._waiting:
                return None

            return self._result

        def wait(self, duration=None):
            t = self.waitAsync(duration)
            t.join()

        def waitAsync(self, duration=None):
            thread = Thread(target=self._wait_for_finished, args=(duration,))
            thread.setDaemon(True)
            thread.start()
            return thread

        def _wait_for_finished(self, duration):
            self._waiting = True
            if duration is None:
                self._client.wait_for_result()
            else:
                self._client.wait_for_result(rospy.Duration(duration))

            self._result = self._client.get_state()
            self._waiting = False

        def cancel(self):
            self._client.cancel_all_goals()

class RosSubscriber(object):

    def __init__(self, topic, dataType, idleTime=15):
        self._lastAccess = time.time()
        self._subscriber = None
        self._topic = topic
        self._dataType = dataType
        self._newMessage = False
        self._idleTimeout = idleTime
        self._data = None

    @property
    def topic(self):
        return self._topic

    @property
    def hasNewMessage(self):
        self._touch()
        return self._newMessage

    @property
    def lastMessage(self):
        self._touch()
        self._newMessage = False
        return self._data

    def _touch(self):
        self._lastAccess = time.time()
        if self._subscriber == None:
            self._subscriber = rospy.Subscriber(self._topic, self._dataType, self._callback)

    def unregister(self):
        if self._subscriber != None:
            self._subscriber.unregister()
            self._subscriber = None

    def _callback(self, msg):
        self._data = msg

        self._newMessage = True
        if time.time() - self._lastAccess > self._idleTimeout:
            self.unregister()

if __name__ == '__main__':

    rospy.init_node('sf_controller')
    SunflowerAction(rospy.get_name())
    rospy.spin()
<commit_msg>fix api bug<commit_after>#!/usr/bin/env python

'''
Created on 12 Mar 2013

@author: nathan
'''
import roslib
roslib.load_manifest('sf_controller')

import time
import math
from threading import Thread

import rospy
import actionlib
import sf_controller_msgs.msg
import sf_lights_msgs.msg
from dynamixel_msgs.msg import JointState
from geometry_msgs.msg import PoseStamped, Twist
from p2os_driver.msg import MotorState
from tf.transformations import quaternion_from_euler
from std_msgs.msg import Float64
from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal

class SunflowerAction(object):

    _actionHandles = {}

    def __init__(self, name):
        self._action_name = name
        self._as = actionlib.SimpleActionServer(self._action_name, sf_controller_msgs.msg.SunflowerAction, execute_cb=self.execute_cb, auto_start=False)
        self._as.start()
        rospy.loginfo("Started Sunflower Controller ActionServer")
        self._pubs = self._connectToJoints()
        self._subs = {}
        (self._cmdVel, self._motorState) = self._connectToWheels()
        self._feedback = sf_controller_msgs.msg.SunflowerFeedback()
        self._result = sf_controller_msgs.msg.SunflowerResult()

    def __del__(self):
        for pub in self._pubs:
            pub.unregister()

    def park(self):
        g = sf_controller_msgs.msg.SunflowerAction()
        g.component = 'head'
        g.positions = [0, 0, -1.67, 1.67]
        self.execute_cb(g)
        g = sf_controller_msgs.msg.SunflowerAction()
        g.component = 'tray'
        g.positions = [0, ]
        self.execute_cb(g)
        g = sf_controller_msgs.msg.SunflowerAction()
        g.component = 'lights'
        g.positions = [0, 0, 0]
        self.execute_cb(g)

    def _connectToWheels(self):
        return (rospy.Publisher('cmd_vel', Twist), rospy.Publisher('cmd_motor_state', MotorState))

    def _connectToJoints(self):
        pubs = {}
        joints = rospy.get_param('/sf_controller', None)
        if joints:
            for value in joints.values():
                if 'joint_names' in value:
                    for jointName in value['joint_names']:
                        topic = jointName + '_controller'
                        if topic not in pubs:
                            pubs[topic] = rospy.Publisher(topic + '/command', Float64)
        else:
            rospy.logerr("%s:  No joints received from parameter server",
                          self._action_name)

        return pubs

    def execute_cb(self, goal):
        if goal.component == 'light':
            result = self.setlight(goal.jointPositions)
        elif goal.action == 'move':
            result = self.move(goal)
        elif goal.action == 'init':
            result = self.init(goal.component)
        elif goal.action == 'stop':
            self.stop(goal.component)
            result = True
        elif goal.action == 'park':
            self.park()
            result = True
        else:
            rospy.logwarn("Unknown action %s", goal.action)
            self._result.result = -1
            self._as.set_aborted(self._result)

        if result:
            self._result.result = 0
            self._as.set_succeeded(self._result)
        else:
            self._result.result = -1
            self._as.set_aborted(self._result)

    def init(self, name):
        if name == 'base' or name == 'base_direct':
            self._motorState.publish(MotorState(1))

        self._as.set_succeeded(self._result)

    def stop(self, name):
        rospy.loginfo("%s: Stopping %s",
            self._action_name,
            name)
        if name == 'base':
            client = actionlib.SimpleActionClient('/lights', sf_lights_msgs.msg.LightsAction)
            client.wait_for_server()
            client.cancel_all_goals()
        elif name == 'light':
            client = actionlib.SimpleActionClient('/move_base', MoveBaseAction)
            client.wait_for_server()
            client.cancel_all_goals()
        else:
            pass

        self._as.set_succeeded(self._result)

    def setlight(self, color):
        client = actionlib.SimpleActionClient('/lights', sf_lights_msgs.msg.LightsAction)
        client.wait_for_server()
        goal = sf_lights_msgs.msg.LightsGoal(rgb=color)
        handle = _ActionHandle(client)
        client.send_goal(goal)
        handle.wait()
        handle.result

    def move(self, goal):
        joints = goal.jointPositions

        if(goal.namedPosition != '' and goal.namedPosition != None):
            param = '/sf_controller/' + goal.component + '/' + goal.namedPosition
            if(rospy.has_param(param)):
                joints = rospy.get_param(param)[0]

        rospy.loginfo("%s: Setting %s to %s",
                self._action_name,
                goal.component,
                goal.namedPosition or joints)

        if goal.component == 'base':
            result = self.navigate(goal, joints)
        elif goal.component == 'base_direct':
            result = self.moveBase(goal, joints)
        else:
            result = self.moveJoints(goal, joints)

        rospy.logdebug("%s: '%s to %s' Result:%s",
                self._action_name,
                goal.component,
                goal.namedPosition or joints,
                result)

        return result == 3

    def moveBase(self, goal, positions):
        # TODO: These should be in a config file
        maxTrans = 1.5
        LINEAR_RATE = 0.3  # [m/s]
        maxRot = 2 * math.pi
        ROTATION_RATE = 0.5  # [rad/s]

        rotation = positions[0]
        linear = positions[1]

        rospy.loginfo("Moving base %s rad and %s meters", rotation, linear)

        # step 0: check validity of parameters:
        if not isinstance(rotation, (int, float)) or not isinstance(linear, (int, float)):
                rospy.logerr("Non-numeric rotation list, aborting moveBase")
                return False
        if abs(linear) >= maxTrans:
                rospy.logerr("Maximal relative translation step exceeded(max: %sm), aborting moveBase" % maxTrans)
                return False
        if abs(rotation) >= maxRot:
                rospy.logerr("Maximal relative rotation step exceeded(max: %srad), aborting moveBase" % maxRot)
                return False

        # step 1: determine duration of motion so that upper thresholds for both translational as well as rotational velocity are not exceeded
        duration_trans_sec = abs(linear) / LINEAR_RATE
        duration_rot_sec = abs(rotation) / ROTATION_RATE
        duration_sec = max(duration_trans_sec, duration_rot_sec)

        # step 2: determine actual velocities based on calculated duration
        x_vel = linear / duration_sec
        rot_vel = rotation / duration_sec

        # step 3: send constant velocity command to base_controller for the calculated duration of motion
        # pub = rospy.Publisher('cmd_vel', Twist)
        twist = Twist()
        twist.linear.x = x_vel
        twist.angular.z = rot_vel
        duration_ros = rospy.Duration.from_sec(duration_sec)  # duration of motion in ROS time

        r = rospy.Rate(50)
        end_time = rospy.Time.now() + duration_ros
        self._cmdVel.publish(twist)
        while not rospy.is_shutdown() and rospy.Time.now() < end_time:
            # pub.publish(twist) #p2os has issues if you republish, seems to continue using last received cmd_vel
            if self._as.is_preempt_requested():
                rospy.loginfo('%s: Preempted' % self._action_name)
                self._as.set_preempted()
                return 2
            r.sleep()

        self._cmdVel.publish(Twist())  # send a stop command, see above comment

        return 3

    def navigate(self, goal, positions):
        pose = PoseStamped()
        pose.header.stamp = rospy.Time.now()
        pose.header.frame_id = "/map"
        pose.pose.position.x = positions[0]
        pose.pose.position.y = positions[1]
        pose.pose.position.z = 0.0
        q = quaternion_from_euler(0, 0, positions[2])
        pose.pose.orientation.x = q[0]
        pose.pose.orientation.y = q[1]
        pose.pose.orientation.z = q[2]
        pose.pose.orientation.w = q[3]

        client = actionlib.SimpleActionClient('/move_base', MoveBaseAction)
        client_goal = MoveBaseGoal()
        client_goal.target_pose = pose

        client.wait_for_server()
        rospy.loginfo("%s: Navigating to %s",
                self._action_name,
                pose)

        handle = _ActionHandle(client)
        client.send_goal(client_goal)
        handle.wait()
        return handle.result

    def moveJoints(self, goal, positions):
        subs = []
        pubs = {}

        component_name = '/sf_controller/' + goal.component
        try:
            joint_names = rospy.get_param(component_name + '/joint_names')
        except KeyError:
            # assume component is a named joint
            joint_names = [goal.component, ]
        for i in range(0, len(joint_names)):
            topic = joint_names[i] + '_controller'
            if topic not in self._pubs:
                rospy.logerr('Undefined joint controller %s', topic)
                return False

            subs.append(RosSubscriber(topic + '/state', JointState))
            pubs[topic] = self._pubs[topic]
            self._pubs[topic].publish(Float64(positions[i]))

        handle = _SubscriberHandle(subs, pubs)
        handle.wait()
        return handle.result

#------------------- action_handle section -------------------#
# Subscriber handle class.
#
# The subscriber handle is used to implement asynchronous behaviour within the script.
class _SubscriberHandle(object):
        # Initialises the action handle.
        def __init__(self, subscribers, publishers):
            self._subscribers = subscribers
            self._publishers = publishers
            self._waiting = False
            self._result = None

        @property
        def result(self):
            if self._waiting:
                return None

            return self._result

        def wait(self, duration=None):
            t = self.waitAsync(duration)
            t.join()

        def waitAsync(self, duration=None):
            thread = Thread(target=self._wait_for_finished, args=(duration,))
            thread.setDaemon(True)
            thread.start()
            return thread

        def _wait_for_finished(self, duration):
            self._waiting = True
            reached = True
            while True:
                r = rospy.Rate(100)
                for sub in self._subscribers:
                    if rospy.is_shutdown():
                        return False

                    while not sub.hasNewMessage:
                        r.sleep()

                    if sub.lastMessage.is_moving:
                        continue

                    reached &= abs(sub.lastMessage.error) < 0.03
                    if not reached:
                        rospy.logwarn("%s unable to reach %s.  Error value: %s" % (sub.topic, sub.lastMessage.goal_pos, sub.lastMessage.error))

                    self._subscribers.remove(sub)

                if not self._subscribers:
                    break

            self._result = 3 if reached else 4

        def cancel(self):
            for sub in self._subscribers:
                self._publishers[sub.topic].publish(Float64(sub.lastMessage.current_pos))

#------------------- action_handle section -------------------#
# # Action handle class.
#
# The action handle is used to implement asynchronous behaviour within the script.
class _ActionHandle(object):
        # Initialises the action handle.
        def __init__(self, simpleActionClient):
            self._client = simpleActionClient
            self._waiting = False
            self._result = None

        @property
        def result(self):
            if self._waiting:
                return None

            return self._result

        def wait(self, duration=None):
            t = self.waitAsync(duration)
            t.join()

        def waitAsync(self, duration=None):
            thread = Thread(target=self._wait_for_finished, args=(duration,))
            thread.setDaemon(True)
            thread.start()
            return thread

        def _wait_for_finished(self, duration):
            self._waiting = True
            if duration is None:
                self._client.wait_for_result()
            else:
                self._client.wait_for_result(rospy.Duration(duration))

            self._result = self._client.get_state()
            self._waiting = False

        def cancel(self):
            self._client.cancel_all_goals()

class RosSubscriber(object):

    def __init__(self, topic, dataType, idleTime=15):
        self._lastAccess = time.time()
        self._subscriber = None
        self._topic = topic
        self._dataType = dataType
        self._newMessage = False
        self._idleTimeout = idleTime
        self._data = None

    @property
    def topic(self):
        return self._topic

    @property
    def hasNewMessage(self):
        self._touch()
        return self._newMessage

    @property
    def lastMessage(self):
        self._touch()
        self._newMessage = False
        return self._data

    def _touch(self):
        self._lastAccess = time.time()
        if self._subscriber == None:
            self._subscriber = rospy.Subscriber(self._topic, self._dataType, self._callback)

    def unregister(self):
        if self._subscriber != None:
            self._subscriber.unregister()
            self._subscriber = None

    def _callback(self, msg):
        self._data = msg

        self._newMessage = True
        if time.time() - self._lastAccess > self._idleTimeout:
            self.unregister()

if __name__ == '__main__':

    rospy.init_node('sf_controller')
    SunflowerAction(rospy.get_name())
    rospy.spin()
<|endoftext|>


<commit_before># Copyright 2020 Valentin Vinagre <valentin.vinagre@sygel.es>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

{
    "name": "Mail Show Follower",
    "summary": "Show CC document followers in mails.",
    "version": "12.0.1.1.0",
    "category": "Mail",
    "website": "https://github.com/OCA/social",
    "author": "Sygel, Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "application": False,
    "installable": True,
    "depends": [
        "base",
        "mail"
    ],
    "data": [
        "views/res_config_settings.xml",
        "views/res_users.xml"
    ],
}
<commit_msg>Update mail_show_follower/__manifest__.py<commit_after># Copyright 2020 Valentin Vinagre <valentin.vinagre@sygel.es>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

{
    "name": "Mail Show Follower",
    "summary": "Show CC document followers in mails.",
    "version": "12.0.1.0.1",
    "category": "Mail",
    "website": "https://github.com/OCA/social",
    "author": "Sygel, Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "application": False,
    "installable": True,
    "depends": [
        "base",
        "mail"
    ],
    "data": [
        "views/res_config_settings.xml",
        "views/res_users.xml"
    ],
}
<|endoftext|>


<commit_before># Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Google ID Token helpers.

Provides support for verifying `OpenID Connect ID Tokens`_, especially ones
generated by Google infrastructure.

To parse and verify an ID Token issued by Google's OAuth 2.0 authorization
server use :func:`verify_oauth2_token`. To verify an ID Token issued by
Firebase, use :func:`verify_firebase_token`.

A general purpose ID Token verifier is available as :func:`verify_token`.

Example::

    from google.oauth2 import id_token
    from google.auth.transport import requests

    request = requests.Request()

    id_info = id_token.verify_oauth2_token(
        token, request, 'my-client-id.example.com')

    if id_info['iss'] != 'https://accounts.google.com':
        raise ValueError('Wrong issuer.')

    userid = id_info['sub']

By default, this will re-fetch certificates for each verification. Because
Google's public keys are only changed infrequently (on the order of once per
day), you may wish to take advantage of caching to reduce latency and the
potential for network errors. This can be accomplished using an external
library like `CacheControl`_ to create a cache-aware
:class:`google.auth.transport.Request`::

    import cachecontrol
    import google.auth.transport.requests
    import requests

    session = requests.session()
    cached_session = cachecontrol.CacheControl(session)
    request = google.auth.transport.requests.Request(session=cached_session)

.. _OpenID Connect ID Token:
    http://openid.net/specs/openid-connect-core-1_0.html#IDToken
.. _CacheControl: https://cachecontrol.readthedocs.io
"""

import json
import os

import six
from six.moves import http_client

from google.auth import environment_vars
from google.auth import exceptions
from google.auth import jwt

# The URL that provides public certificates for verifying ID tokens issued
# by Google's OAuth 2.0 authorization server.
_GOOGLE_OAUTH2_CERTS_URL = "https://www.googleapis.com/oauth2/v1/certs"

# The URL that provides public certificates for verifying ID tokens issued
# by Firebase and the Google APIs infrastructure
_GOOGLE_APIS_CERTS_URL = (
    "https://www.googleapis.com/robot/v1/metadata/x509"
    "/securetoken@system.gserviceaccount.com"
)

def _fetch_certs(request, certs_url):
    """Fetches certificates.

    Google-style cerificate endpoints return JSON in the format of
    ``{'key id': 'x509 certificate'}``.

    Args:
        request (google.auth.transport.Request): The object used to make
            HTTP requests.
        certs_url (str): The certificate endpoint URL.

    Returns:
        Mapping[str, str]: A mapping of public key ID to x.509 certificate
            data.
    """
    response = request(certs_url, method="GET")

    if response.status != http_client.OK:
        raise exceptions.TransportError(
            "Could not fetch certificates at {}".format(certs_url)
        )

    return json.loads(response.data.decode("utf-8"))

def verify_token(id_token, request, audience=None, certs_url=_GOOGLE_OAUTH2_CERTS_URL):
    """Verifies an ID token and returns the decoded token.

    Args:
        id_token (Union[str, bytes]): The encoded token.
        request (google.auth.transport.Request): The object used to make
            HTTP requests.
        audience (str): The audience that this token is intended for. If None
            then the audience is not verified.
        certs_url (str): The URL that specifies the certificates to use to
            verify the token. This URL should return JSON in the format of
            ``{'key id': 'x509 certificate'}``.

    Returns:
        Mapping[str, Any]: The decoded token.
    """
    certs = _fetch_certs(request, certs_url)

    return jwt.decode(id_token, certs=certs, audience=audience)

def verify_oauth2_token(id_token, request, audience=None):
    """Verifies an ID Token issued by Google's OAuth 2.0 authorization server.

    Args:
        id_token (Union[str, bytes]): The encoded token.
        request (google.auth.transport.Request): The object used to make
            HTTP requests.
        audience (str): The audience that this token is intended for. This is
            typically your application's OAuth 2.0 client ID. If None then the
            audience is not verified.

    Returns:
        Mapping[str, Any]: The decoded token.
    """
    return verify_token(
        id_token, request, audience=audience, certs_url=_GOOGLE_OAUTH2_CERTS_URL
    )

def verify_firebase_token(id_token, request, audience=None):
    """Verifies an ID Token issued by Firebase Authentication.

    Args:
        id_token (Union[str, bytes]): The encoded token.
        request (google.auth.transport.Request): The object used to make
            HTTP requests.
        audience (str): The audience that this token is intended for. This is
            typically your Firebase application ID. If None then the audience
            is not verified.

    Returns:
        Mapping[str, Any]: The decoded token.
    """
    return verify_token(
        id_token, request, audience=audience, certs_url=_GOOGLE_APIS_CERTS_URL
    )

def fetch_id_token(request, audience):
    """Fetch the ID Token from the current environment.

    This function acquires ID token from the environment in the following order:

    1. If the application is running in Compute Engine, App Engine or Cloud Run,
       then the ID token are obtained from the metadata server.
    2. If the environment variable ``GOOGLE_APPLICATION_CREDENTIALS`` is set
       to the path of a valid service account JSON file, then ID token is
       acquired using this service account credentials.
    3. If metadata server doesn't exist and no valid service account credentials
       are found, :class:`~google.auth.exceptions.DefaultCredentialsError` will
       be raised.

    Example::

        import google.oauth2.id_token
        import google.auth.transport.requests

        request = google.auth.transport.requests.Request()
        target_audience = "https://pubsub.googleapis.com"

        id_token = google.oauth2.id_token.fetch_id_token(request, target_audience)

    Args:
        request (google.auth.transport.Request): A callable used to make
            HTTP requests.
        audience (str): The audience that this ID token is intended for.

    Returns:
        str: The ID token.

    Raises:
        ~google.auth.exceptions.DefaultCredentialsError:
            If metadata server doesn't exist and no valid service account
            credentials are found.
    """
    # 1. First try to fetch ID token from metada server if it exists. The code
    # works for GAE and Cloud Run metadata server as well.
    try:
        from google.auth import compute_engine

        credentials = compute_engine.IDTokenCredentials(
            request, audience, use_metadata_identity_endpoint=True
        )
        credentials.refresh(request)
        return credentials.token
    except (ImportError, exceptions.TransportError):
        pass

    # 2. Try to use service account credentials to get ID token.

    # Try to get credentials from the GOOGLE_APPLICATION_CREDENTIALS environment
    # variable.
    credentials_filename = os.environ.get(environment_vars.CREDENTIALS)
    if not (
        credentials_filename
        and os.path.exists(credentials_filename)
        and os.path.isfile(credentials_filename)
    ):
        raise exceptions.DefaultCredentialsError(
            "Neither metadata server or valid service account credentials are found."
        )

    try:
        with open(credentials_filename, "r") as f:
            info = json.load(f)
            credentials_content = (
                (info.get("type") == "service_account") and info or None
            )

            from google.oauth2 import service_account

            credentials = service_account.IDTokenCredentials.from_service_account_info(
                credentials_content, target_audience=audience
            )
    except ValueError as caught_exc:
        new_exc = exceptions.DefaultCredentialsError(
            "Neither metadata server or valid service account credentials are found.",
            caught_exc,
        )
        six.raise_from(new_exc, caught_exc)

    credentials.refresh(request)
    return credentials.token
<commit_msg>fix: catch exceptions.RefreshError (#508)<commit_after># Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Google ID Token helpers.

Provides support for verifying `OpenID Connect ID Tokens`_, especially ones
generated by Google infrastructure.

To parse and verify an ID Token issued by Google's OAuth 2.0 authorization
server use :func:`verify_oauth2_token`. To verify an ID Token issued by
Firebase, use :func:`verify_firebase_token`.

A general purpose ID Token verifier is available as :func:`verify_token`.

Example::

    from google.oauth2 import id_token
    from google.auth.transport import requests

    request = requests.Request()

    id_info = id_token.verify_oauth2_token(
        token, request, 'my-client-id.example.com')

    if id_info['iss'] != 'https://accounts.google.com':
        raise ValueError('Wrong issuer.')

    userid = id_info['sub']

By default, this will re-fetch certificates for each verification. Because
Google's public keys are only changed infrequently (on the order of once per
day), you may wish to take advantage of caching to reduce latency and the
potential for network errors. This can be accomplished using an external
library like `CacheControl`_ to create a cache-aware
:class:`google.auth.transport.Request`::

    import cachecontrol
    import google.auth.transport.requests
    import requests

    session = requests.session()
    cached_session = cachecontrol.CacheControl(session)
    request = google.auth.transport.requests.Request(session=cached_session)

.. _OpenID Connect ID Token:
    http://openid.net/specs/openid-connect-core-1_0.html#IDToken
.. _CacheControl: https://cachecontrol.readthedocs.io
"""

import json
import os

import six
from six.moves import http_client

from google.auth import environment_vars
from google.auth import exceptions
from google.auth import jwt

# The URL that provides public certificates for verifying ID tokens issued
# by Google's OAuth 2.0 authorization server.
_GOOGLE_OAUTH2_CERTS_URL = "https://www.googleapis.com/oauth2/v1/certs"

# The URL that provides public certificates for verifying ID tokens issued
# by Firebase and the Google APIs infrastructure
_GOOGLE_APIS_CERTS_URL = (
    "https://www.googleapis.com/robot/v1/metadata/x509"
    "/securetoken@system.gserviceaccount.com"
)

def _fetch_certs(request, certs_url):
    """Fetches certificates.

    Google-style cerificate endpoints return JSON in the format of
    ``{'key id': 'x509 certificate'}``.

    Args:
        request (google.auth.transport.Request): The object used to make
            HTTP requests.
        certs_url (str): The certificate endpoint URL.

    Returns:
        Mapping[str, str]: A mapping of public key ID to x.509 certificate
            data.
    """
    response = request(certs_url, method="GET")

    if response.status != http_client.OK:
        raise exceptions.TransportError(
            "Could not fetch certificates at {}".format(certs_url)
        )

    return json.loads(response.data.decode("utf-8"))

def verify_token(id_token, request, audience=None, certs_url=_GOOGLE_OAUTH2_CERTS_URL):
    """Verifies an ID token and returns the decoded token.

    Args:
        id_token (Union[str, bytes]): The encoded token.
        request (google.auth.transport.Request): The object used to make
            HTTP requests.
        audience (str): The audience that this token is intended for. If None
            then the audience is not verified.
        certs_url (str): The URL that specifies the certificates to use to
            verify the token. This URL should return JSON in the format of
            ``{'key id': 'x509 certificate'}``.

    Returns:
        Mapping[str, Any]: The decoded token.
    """
    certs = _fetch_certs(request, certs_url)

    return jwt.decode(id_token, certs=certs, audience=audience)

def verify_oauth2_token(id_token, request, audience=None):
    """Verifies an ID Token issued by Google's OAuth 2.0 authorization server.

    Args:
        id_token (Union[str, bytes]): The encoded token.
        request (google.auth.transport.Request): The object used to make
            HTTP requests.
        audience (str): The audience that this token is intended for. This is
            typically your application's OAuth 2.0 client ID. If None then the
            audience is not verified.

    Returns:
        Mapping[str, Any]: The decoded token.
    """
    return verify_token(
        id_token, request, audience=audience, certs_url=_GOOGLE_OAUTH2_CERTS_URL
    )

def verify_firebase_token(id_token, request, audience=None):
    """Verifies an ID Token issued by Firebase Authentication.

    Args:
        id_token (Union[str, bytes]): The encoded token.
        request (google.auth.transport.Request): The object used to make
            HTTP requests.
        audience (str): The audience that this token is intended for. This is
            typically your Firebase application ID. If None then the audience
            is not verified.

    Returns:
        Mapping[str, Any]: The decoded token.
    """
    return verify_token(
        id_token, request, audience=audience, certs_url=_GOOGLE_APIS_CERTS_URL
    )

def fetch_id_token(request, audience):
    """Fetch the ID Token from the current environment.

    This function acquires ID token from the environment in the following order:

    1. If the application is running in Compute Engine, App Engine or Cloud Run,
       then the ID token are obtained from the metadata server.
    2. If the environment variable ``GOOGLE_APPLICATION_CREDENTIALS`` is set
       to the path of a valid service account JSON file, then ID token is
       acquired using this service account credentials.
    3. If metadata server doesn't exist and no valid service account credentials
       are found, :class:`~google.auth.exceptions.DefaultCredentialsError` will
       be raised.

    Example::

        import google.oauth2.id_token
        import google.auth.transport.requests

        request = google.auth.transport.requests.Request()
        target_audience = "https://pubsub.googleapis.com"

        id_token = google.oauth2.id_token.fetch_id_token(request, target_audience)

    Args:
        request (google.auth.transport.Request): A callable used to make
            HTTP requests.
        audience (str): The audience that this ID token is intended for.

    Returns:
        str: The ID token.

    Raises:
        ~google.auth.exceptions.DefaultCredentialsError:
            If metadata server doesn't exist and no valid service account
            credentials are found.
    """
    # 1. First try to fetch ID token from metada server if it exists. The code
    # works for GAE and Cloud Run metadata server as well.
    try:
        from google.auth import compute_engine

        credentials = compute_engine.IDTokenCredentials(
            request, audience, use_metadata_identity_endpoint=True
        )
        credentials.refresh(request)
        return credentials.token
    except (ImportError, exceptions.TransportError, exceptions.RefreshError):
        pass

    # 2. Try to use service account credentials to get ID token.

    # Try to get credentials from the GOOGLE_APPLICATION_CREDENTIALS environment
    # variable.
    credentials_filename = os.environ.get(environment_vars.CREDENTIALS)
    if not (
        credentials_filename
        and os.path.exists(credentials_filename)
        and os.path.isfile(credentials_filename)
    ):
        raise exceptions.DefaultCredentialsError(
            "Neither metadata server or valid service account credentials are found."
        )

    try:
        with open(credentials_filename, "r") as f:
            info = json.load(f)
            credentials_content = (
                (info.get("type") == "service_account") and info or None
            )

            from google.oauth2 import service_account

            credentials = service_account.IDTokenCredentials.from_service_account_info(
                credentials_content, target_audience=audience
            )
    except ValueError as caught_exc:
        new_exc = exceptions.DefaultCredentialsError(
            "Neither metadata server or valid service account credentials are found.",
            caught_exc,
        )
        six.raise_from(new_exc, caught_exc)

    credentials.refresh(request)
    return credentials.token
<|endoftext|>


<commit_before>import numpy as np
import scipy as sp
from ._model import Model
from ..utils import safe_isinstance, record_import_error
from ._pt_text_generation import PTTextGeneration
from ._tf_text_generation import TFTextGeneration

try:
    import torch
except ImportError as e:
    record_import_error("torch", "Torch could not be imported!", e)

try:
    import tensorflow as tf
except ImportError as e:
    record_import_error("tensorflow", "TensorFlow could not be imported!", e)

class TeacherForcingLogits(Model):
    def __init__(self, model, tokenizer=None, generation_function_for_target_sentence_ids=None, similarity_model=None, similarity_tokenizer=None, device=None):
        """ Generates scores (log odds) for output text explanation algorithms.

        This class supports generation of log odds for transformer models as well as functions. It also provides
        functionality to score custom output text by passing the generation_function_for_target_sentence_ids. In model agnostic
        cases (model is function) it expects a similarity_model and similarity_tokenizer to approximate log odd scores
        for target sentence generated by the model.

        Parameters
        ----------
        model: object or function
            A object of any pretrained transformer model or function which is to be explained.

        tokenizer: object
            A tokenizer object(PreTrainedTokenizer/PreTrainedTokenizerFast) which is used to tokenize source and target sentence.

        generation_function_for_target_sentence_ids: function
            A function which is used to generate custom target ids. Log odds will be generated for these custom target ids.

        similarity_model: object
            A pretrained transformer model object which is used in model agnostic scenario to approximate log odds.

        similarity_tokenizer: object
            A tokenizer object(PreTrainedTokenizer/PreTrainedTokenizerFast) which is used to tokenize sentence in model agnostic scenario.

        device: "cpu" or "cuda" or None
            By default, it infers if system has a gpu and accordingly sets device. Should be 'cpu' or 'gpu'.

        Returns
        -------
        numpy.array
            The scores (log odds) of generating target sentence ids using the model.
        """
        super(TeacherForcingLogits, self).__init__(model)

        #self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device
        self.tokenizer = tokenizer
        self.device = device
        # assign text generation function
        if safe_isinstance(model,"transformers.PreTrainedModel") or safe_isinstance(model,"transformers.TFPreTrainedModel"):
            if generation_function_for_target_sentence_ids is None:
                if safe_isinstance(model,"transformers.PreTrainedModel"):
                    self.generation_function_for_target_sentence_ids = PTTextGeneration(self.model, tokenizer=self.tokenizer, device=self.device)
                elif safe_isinstance(model,"transformers.TFPreTrainedModel"):
                    self.generation_function_for_target_sentence_ids = TFTextGeneration(self.model, tokenizer=self.tokenizer, device=self.device)
            else:
                self.generation_function_for_target_sentence_ids = generation_function_for_target_sentence_ids
            self.model_agnostic = False
            #self.model = self.to_device(model)
            self.similarity_model = model
            self.similarity_tokenizer = tokenizer
        else:
            if generation_function_for_target_sentence_ids is None:
                if safe_isinstance(model,"transformers.PreTrainedModel"):
                    self.generation_function_for_target_sentence_ids = PTTextGeneration(self.model, similarity_tokenizer=similarity_tokenizer, device=self.device)
                elif safe_isinstance(model,"transformers.TFPreTrainedModel"):
                    self.generation_function_for_target_sentence_ids = TFTextGeneration(self.model, similarity_tokenizer=similarity_tokenizer, device=self.device)
            else:
                self.generation_function_for_target_sentence_ids = generation_function_for_target_sentence_ids
            #self.similarity_model = self.to_device(similarity_model)
            self.similarity_model = similarity_model
            self.similarity_tokenizer = similarity_tokenizer
            self.model_agnostic = True
        # initializing X which is the original input for every new row of explanation
        self.X = None
        self.target_sentence_ids = None
        self.output_names = None

    def __call__(self, masked_X, X):
        """ Computes log odds scores from a given batch of masked input and original input for text/image.

        Parameters
        ----------
        masked_X: numpy.array
            An array containing a list of masked inputs.

        X: numpy.array
            An array containing a list of original inputs

        Returns
        -------
        numpy.array
            A numpy array of log odds scores for every input pair (masked_X, X)
        """
        output_batch=[]
        for masked_x, x in zip(masked_X, X):
            # update target sentence ids and original input for a new explanation row
            self.update_cache_X(x)
            # pass the masked input from which to generate source sentence ids
            source_sentence_ids = self.get_source_sentence_ids(masked_x)
            logits = self.get_teacher_forced_logits(source_sentence_ids, self.target_sentence_ids)
            logodds = self.get_logodds(logits)
            output_batch.append(logodds)
        return np.array(output_batch)

    def update_cache_X(self, X):
        """ The function updates original input(X) and target sentence ids.

        It mimics the caching mechanism to update the original input and target sentence ids
        that are to be explained and which updates for every new row of explanation.

        Parameters
        ----------
        X: string or numpy.array
            Input(Text/Image) for an explanation row.
        """
        # check if the source sentence has been updated (occurs when explaining a new row)
        if (self.X is None) or (isinstance(self.X, np.ndarray) and (self.X != X).all()) or (isinstance(self.X, str) and (self.X != X)):
            self.X = X
            self.output_names = self.get_output_names_and_update_target_sentence_ids(self.X)

    def get_output_names_and_update_target_sentence_ids(self, X):
        """ Gets the output tokens from input(X) by computing the
            target sentence ids using the using the generation_function_for_target_sentence_ids()
            and next getting output names using the similarity_tokenizer.

        Parameters
        ----------
        X: string or numpy array
            Input(Text/Image) for an explanation row.

        Returns
        -------
        list
            A list of output tokens.
        """
        self.target_sentence_ids = self.generation_function_for_target_sentence_ids(X)
        return self.similarity_tokenizer.convert_ids_to_tokens(self.target_sentence_ids[0,:])<commit_msg>Added abstract functions in TeacherForcingLogits class<commit_after>import numpy as np
import scipy as sp
from ._model import Model
from ..utils import safe_isinstance, record_import_error
from ._pt_text_generation import PTTextGeneration
from ._tf_text_generation import TFTextGeneration

try:
    import torch
except ImportError as e:
    record_import_error("torch", "Torch could not be imported!", e)

try:
    import tensorflow as tf
except ImportError as e:
    record_import_error("tensorflow", "TensorFlow could not be imported!", e)

class TeacherForcingLogits(Model):
    def __init__(self, model, tokenizer=None, generation_function_for_target_sentence_ids=None, similarity_model=None, similarity_tokenizer=None, device=None):
        """ Generates scores (log odds) for output text explanation algorithms.

        This class supports generation of log odds for transformer models as well as functions. It also provides
        functionality to score custom output text by passing the generation_function_for_target_sentence_ids. In model agnostic
        cases (model is function) it expects a similarity_model and similarity_tokenizer to approximate log odd scores
        for target sentence generated by the model.

        Parameters
        ----------
        model: object or function
            A object of any pretrained transformer model or function which is to be explained.

        tokenizer: object
            A tokenizer object(PreTrainedTokenizer/PreTrainedTokenizerFast) which is used to tokenize source and target sentence.

        generation_function_for_target_sentence_ids: function
            A function which is used to generate custom target ids. Log odds will be generated for these custom target ids.

        similarity_model: object
            A pretrained transformer model object which is used in model agnostic scenario to approximate log odds.

        similarity_tokenizer: object
            A tokenizer object(PreTrainedTokenizer/PreTrainedTokenizerFast) which is used to tokenize sentence in model agnostic scenario.

        device: "cpu" or "cuda" or None
            By default, it infers if system has a gpu and accordingly sets device. Should be 'cpu' or 'gpu'.

        Returns
        -------
        numpy.array
            The scores (log odds) of generating target sentence ids using the model.
        """
        super(TeacherForcingLogits, self).__init__(model)

        #self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device
        self.tokenizer = tokenizer
        self.device = device
        # assign text generation function
        if safe_isinstance(model,"transformers.PreTrainedModel") or safe_isinstance(model,"transformers.TFPreTrainedModel"):
            if generation_function_for_target_sentence_ids is None:
                if safe_isinstance(model,"transformers.PreTrainedModel"):
                    self.generation_function_for_target_sentence_ids = PTTextGeneration(self.model, tokenizer=self.tokenizer, device=self.device)
                elif safe_isinstance(model,"transformers.TFPreTrainedModel"):
                    self.generation_function_for_target_sentence_ids = TFTextGeneration(self.model, tokenizer=self.tokenizer, device=self.device)
            else:
                self.generation_function_for_target_sentence_ids = generation_function_for_target_sentence_ids
            self.model_agnostic = False
            #self.model = self.to_device(model)
            self.similarity_model = model
            self.similarity_tokenizer = tokenizer
        else:
            if generation_function_for_target_sentence_ids is None:
                if safe_isinstance(model,"transformers.PreTrainedModel"):
                    self.generation_function_for_target_sentence_ids = PTTextGeneration(self.model, similarity_tokenizer=similarity_tokenizer, device=self.device)
                elif safe_isinstance(model,"transformers.TFPreTrainedModel"):
                    self.generation_function_for_target_sentence_ids = TFTextGeneration(self.model, similarity_tokenizer=similarity_tokenizer, device=self.device)
            else:
                self.generation_function_for_target_sentence_ids = generation_function_for_target_sentence_ids
            #self.similarity_model = self.to_device(similarity_model)
            self.similarity_model = similarity_model
            self.similarity_tokenizer = similarity_tokenizer
            self.model_agnostic = True
        # initializing X which is the original input for every new row of explanation
        self.X = None
        self.target_sentence_ids = None
        self.output_names = None

    def __call__(self, masked_X, X):
        """ Computes log odds scores from a given batch of masked input and original input for text/image.

        Parameters
        ----------
        masked_X: numpy.array
            An array containing a list of masked inputs.

        X: numpy.array
            An array containing a list of original inputs

        Returns
        -------
        numpy.array
            A numpy array of log odds scores for every input pair (masked_X, X)
        """
        output_batch=[]
        for masked_x, x in zip(masked_X, X):
            # update target sentence ids and original input for a new explanation row
            self.update_cache_X(x)
            # pass the masked input from which to generate source sentence ids
            source_sentence_ids = self.get_source_sentence_ids(masked_x)
            logits = self.get_teacher_forced_logits(source_sentence_ids, self.target_sentence_ids)
            logodds = self.get_logodds(logits)
            output_batch.append(logodds)
        return np.array(output_batch)

    def update_cache_X(self, X):
        """ The function updates original input(X) and target sentence ids.

        It mimics the caching mechanism to update the original input and target sentence ids
        that are to be explained and which updates for every new row of explanation.

        Parameters
        ----------
        X: string or numpy.array
            Input(Text/Image) for an explanation row.
        """
        # check if the source sentence has been updated (occurs when explaining a new row)
        if (self.X is None) or (isinstance(self.X, np.ndarray) and (self.X != X).all()) or (isinstance(self.X, str) and (self.X != X)):
            self.X = X
            self.output_names = self.get_output_names_and_update_target_sentence_ids(self.X)

    def get_output_names_and_update_target_sentence_ids(self, X):
        """ Gets the output tokens from input(X) by computing the
            target sentence ids using the using the generation_function_for_target_sentence_ids()
            and next getting output names using the similarity_tokenizer.

        Parameters
        ----------
        X: string or numpy array
            Input(Text/Image) for an explanation row.

        Returns
        -------
        list
            A list of output tokens.
        """
        self.target_sentence_ids = self.generation_function_for_target_sentence_ids(X)
        return self.similarity_tokenizer.convert_ids_to_tokens(self.target_sentence_ids[0,:])

    def get_source_sentence_ids(self, X):
        """ Implement in subclass.
        """
        pass

    def get_logodds(self, logits):
        """ Implement in subclass.
        """
        pass

    def get_teacher_forced_logits(self,source_sentence_ids,target_sentence_ids):
        """ Implement in subclass.
        """
        pass<|endoftext|>


<commit_before>__author__ = 'sxjscience'

import mxnet as mx
import numpy
from defaults import *

def dqn_loss(qval):
    return numpy.argmax(qval, axis=1)

def dqn_out_grad(qval, action, reward):
    if isinstance(qval, mx.ndarray.NDArray):
        qval = qval.asnumpy()
    action = action.flatten().astype(numpy.int)
    out_grad = numpy.zeros(qval.shape)
    out_grad[numpy.arange(qval.shape[0]), action] = numpy.clip(qval[numpy.arange(action.shape[0]), action] - reward, -1, 1)
    return out_grad

# def dqn_action(qval, exploration_prob=Defaults.EXPLORATION_EPSILON_DECAY):

def dqn_network(action_num=4):
    net = mx.symbol.Variable('data')
    net = mx.symbol.Convolution(data=net, name='conv1', kernel=(8, 8), stride=(4, 4), num_filter=16)
    net = mx.symbol.Activation(data=net, name='relu1', act_type="relu")
    net = mx.symbol.Convolution(data=net, name='conv2', kernel=(4, 4), stride=(2, 2), num_filter=32)
    net = mx.symbol.Activation(data=net, name='relu2', act_type="relu")
    net = mx.symbol.FullyConnected(data=net, name='fc3', num_hidden=256)
    net = mx.symbol.Activation(data=net, name='relu3', act_type="relu")
    net = mx.symbol.FullyConnected(data=net, name='fc4', num_hidden=action_num)
    return net

data_shape = (4, 100, 100)
dqn_sym = dqn_network()
print dqn_sym.list_arguments()
dqn_sym.simple_bind(ctx=get_ctx(), data=data_shape)

<commit_msg>Revised DQN<commit_after>__author__ = 'sxjscience'

import mxnet as mx
import numpy
from defaults import *

def dqn_out_grad(qval, action, reward):
    if isinstance(qval, mx.ndarray.NDArray):
        qval = qval.asnumpy()
    action = action.flatten().astype(numpy.int)
    out_grad = numpy.zeros(qval.shape)
    out_grad[numpy.arange(qval.shape[0]), action] = numpy.clip(qval[numpy.arange(action.shape[0]), action] - reward, -1, 1)
    return out_grad

def dqn_action(qval, exploration_prob=Defaults.EXPLORATION_EPSILON_DECAY):
    rng = get_numpy_rng()
    exploration_prob = rng.choice([0, 1], size=qval.shape[0], p=[.1, .9])

def dqn_network(action_num=4):
    net = mx.symbol.Variable('data')
    net = mx.symbol.Convolution(data=net, name='conv1', kernel=(8, 8), stride=(4, 4), num_filter=16)
    net = mx.symbol.Activation(data=net, name='relu1', act_type="relu")
    net = mx.symbol.Convolution(data=net, name='conv2', kernel=(4, 4), stride=(2, 2), num_filter=32)
    net = mx.symbol.Activation(data=net, name='relu2', act_type="relu")
    net = mx.symbol.Flatten(data=net)
    net = mx.symbol.FullyConnected(data=net, name='fc3', num_hidden=256)
    net = mx.symbol.Activation(data=net, name='relu3', act_type="relu")
    net = mx.symbol.FullyConnected(data=net, name='fc4', num_hidden=action_num)
    return net

data_shape = (4, 100, 100)
dqn_sym = dqn_network()
print dqn_sym.list_arguments()
dqn_sym.simple_bind(ctx=get_ctx(), data=data_shape)

<|endoftext|>


<commit_before><commit_msg>Fixed regex to pick correct items (#42557)<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
===============================================================================
Base classes for geometric regions (:mod:`sknano.utils.geometric_shapes._base`)
===============================================================================

.. currentmodule:: sknano.utils.geometric_shapes._base

"""
from __future__ import absolute_import, division, print_function
__docformat__ = 'restructuredtext en'

from abc import ABCMeta, abstractmethod, abstractproperty

__all__ = ['GeometricRegion']

class GeometricRegion(object):
    """Abstract base class for geometric regions."""
    __metaclass__ = ABCMeta

    @abstractproperty
    def centroid(self):
        """Centroid of 2D geometric region."""
        raise NotImplementedError

    @abstractmethod
    def contains_point(self):
        """Check if point is contained within geometric region."""
        raise NotImplementedError
<commit_msg>adding abstractmethods for translation and rotation transforms<commit_after># -*- coding: utf-8 -*-
"""
===============================================================================
Base classes for geometric regions (:mod:`sknano.utils.geometric_shapes._base`)
===============================================================================

.. currentmodule:: sknano.utils.geometric_shapes._base

"""
from __future__ import absolute_import, division, print_function
__docformat__ = 'restructuredtext en'

from abc import ABCMeta, abstractmethod, abstractproperty

__all__ = ['GeometricRegion']

class GeometricRegion(object):
    """Abstract base class for geometric regions."""
    __metaclass__ = ABCMeta

    @abstractproperty
    def centroid(self):
        """Centroid of geometric region."""
        raise NotImplementedError

    @abstractmethod
    def contains_point(self):
        """Check if point is contained within geometric region."""
        raise NotImplementedError

    @abstractmethod
    def rotate(self):
        """Rotate region about centroid or arbitrary vector."""
        raise NotImplementedError

    @abstractmethod
    def translate(self):
        """Translate region."""
        raise NotImplementedError
<|endoftext|>


<commit_before>
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

import logging
import itertools

import numpy as np
import pandas as pd

from scipy.spatial.distance import squareform, pdist
from scipy.cluster import hierarchy
from scipy import ndimage

from skimage.filter import rank, threshold_otsu
from skimage.morphology import disk, watershed
from skimage.feature import peak_local_max
from skimage.measure import regionprops

from ..io import ObjectsIO
from ..trajectories import Trajectories

log = logging.getLogger(__name__)

__all__ = []

DEFAULT_PARAMETERS = {'segment_method': 'otsu',
                      'correction': 1.,
                      'smooth': 10,
                      'min_radius': 2.,
                      'max_radius': 8.,
                      'num_cells': 8,
                      'nuc_distance':6,
                      'min_z_size': 4
                      }

def nuclei_detector(data_iterator,
                    metadata={},
                    parameters={},
                    parallel=False,
                    verbose=False,
                    mapper=None,
                    show_progress=False):
    '''
    TODO

    Parameters
    ----------
    data_iterator : Python iterator over dataset
        this iterator should yield one z-stack
        (corresponding to a single time point) at each
        iterations
    metadata: dict
        the metadata for the detection.
    parameters : dict
        the parameters for the detection
    parallel : bool
    verbose : bool
    mapper : bool
    show_progress : bool

    Returns
    -------
    trajs : :class:`Trajectories`
    '''

    _parameters = DEFAULT_PARAMETERS.copy()
    _parameters.update(parameters)
    parameters = _parameters

    parameters['min_z_size'] = parameters['min_z_size'] / metadata['PhysicalSizeZ']
    parameters['min_radius'] /= metadata['PhysicalSizeX']
    parameters['max_radius'] /= metadata['PhysicalSizeX']
    parameters['nuc_distance'] /= metadata['PhysicalSizeX']

    arguments = zip(data_iterator, itertools.repeat(parameters))
    if parallel and mapper is not None:
        results = mapper.map(detect_one_stack,
                             arguments)
    else:
        results = map(detect_one_stack,
                      arguments)

    raw_cell_positions = {}
    for i, result in enumerate(results):
        if 'positions' in result:
            raw_cell_positions[i] = result['positions']
    if not len(raw_cell_positions):
        log.warning('No cell detected anywhere')
        return Trajectories.empty_trajs(['x', 'y', 'z', 'w', 'I'])
    nuclei_positions = pd.concat(
        raw_cell_positions,
        keys=raw_cell_positions.keys(),
        names=('t_stamp', 'label'))

    real_times = nuclei_positions.index.get_level_values(
        't_stamp') * metadata['TimeIncrement']
    nuclei_positions['t'] = real_times
    nuclei_positions['t'] = nuclei_positions['t'].astype(np.float)
    nuclei_positions['x'] *=  metadata['PhysicalSizeX']
    nuclei_positions['y'] *=  metadata['PhysicalSizeY']
    nuclei_positions['z'] *=  metadata['PhysicalSizeZ']
    nuclei_positions['w'] *=  metadata['PhysicalSizeX']
    return Trajectories(nuclei_positions)

def detect_one_stack(args, full_output=False):

    z_stack, parameters = args
    if len(z_stack.shape) == 2:
        z_stack = z_stack[np.newaxis, ...]
    labeled_stack = label_stack(z_stack, parameters)
    if labeled_stack is None:
        return {}
    all_props = get_regionprops(labeled_stack,
                                z_stack, parameters)
    if not len(all_props):
        log.warning('No cell found after labelling')
        if full_output:
            output = {'labeled_stack': labeled_stack}
            return output
        else:
            return {}
    if z_stack.shape[0] == 1: ## Only one z plane
        positions = all_props.copy()
        positions.index = pd.Index(range(all_props.shape[0]))
        if full_output:
            output = {'positions': positions,
                      'labeled_stack': labeled_stack}
        else:
            output = {'positions': positions}
        return output

    agregated_props = cluster_regions(all_props, parameters)
    positions = get_cell_positions(agregated_props, parameters)

    if not len(positions):
        output = {'all_props': all_props,
                  'agregated_props': agregated_props,
                  'labeled_stack': labeled_stack}
        return output
    if full_output:
        output = {'positions': positions,
                  'all_props': all_props,
                  'agregated_props': agregated_props,
                  'labeled_stack': labeled_stack}
    else:
        output = {'positions': positions,}
    return output

def label_stack(z_stack, parameters):
    '''
    Apply an automated threshold and a watershed algorithm to
    label cells in each planes of the stack

    Parameters:
    -----------
    z_stack: 3d array of shape (nz, nx, ny)

    Returns:
    --------
    labeled_stack: 3d array of the same shape as z_stack,
    with each detected region labeled
    '''
    segment_method = parameters['segment_method']
    correction = parameters['correction']
    labeled_stack = np.zeros(z_stack.shape, dtype=np.uint8)
    max_int_proj = z_stack.max(axis=0)
    thresh = None

    if segment_method == 'otsu':
        thresh = threshold_otsu(max_int_proj) * correction
    elif segment_method == 'naive':
        thresh = max_int_proj.max() * correction
    else:
        err_string = ('Segmentation method {}'
                      'is not implemented'.format(segment_method))
        raise NotImplementedError(err_string)
    while thresh > z_stack.max():
        log.warning('''Reducing threshold''')
        thresh *= 0.9
    for n, frame in enumerate(z_stack):
        labeled_stack[n] = label_from_thresh(frame, thresh, parameters)
    return labeled_stack

def label_from_thresh(frame, thresh, parameters):

    smooth = parameters['smooth']
    min_distance = np.int(parameters['nuc_distance'])
    image = rank.median(frame.copy(), disk(smooth))
    image = rank.enhance_contrast(image, disk(smooth))
    im_max = image.max()
    if im_max < thresh:
        return np.zeros(image.shape, dtype=np.int32)
    else:
        image = image > thresh
        distance = ndimage.distance_transform_edt(image)
        local_maxi = peak_local_max(distance,
                                    min_distance=min_distance,
                                    indices=False, labels=image)
        markers = ndimage.label(local_maxi)[0]
        return watershed(-distance, markers, mask=image)

def get_regionprops(labeled_stack, z_stack, parameters):
    '''
    Computes the area, centroid position and mean intensity
    of all the regions
    Removes the regions with typical size outside [min_radius, max_radius]

    returns a DataFrame with colmuns ('x', 'y', 'z', 'I', 'w')
    '''

    min_radius = parameters['min_radius']
    max_radius = parameters['max_radius']
    all_props = []
    #properties = ['Area', 'WeightedCentroid', 'MeanIntensity']
    columns = ('x', 'y', 'z', 'I', 'w')
    indices = []
    for z, frame in enumerate(labeled_stack):
        f_prop = regionprops(frame.astype(np.int),# properties,
                             intensity_image=z_stack[z])
        for d in f_prop:
            radius = (d.area / np.pi)**0.5
            if (min_radius  < radius < max_radius):
                all_props.append([d.centroid[0],
                                  d.centroid[1],
                                  z, d.mean_intensity * d.area,
                                  radius])
                indices.append((d.label, z))
    if not len(indices):
        return pd.DataFrame([], index=[])
    indices = pd.MultiIndex.from_tuples(indices, names=('label', 'z'))
    all_props = pd.DataFrame(all_props, index=indices, columns=columns)
    return all_props

def cluster_regions(all_props, parameters):

    radius = parameters['max_radius']
    n_clusters = all_props.shape[0]
    if not n_clusters:
        return None
    if n_clusters == 1:
        log.warning('Only one cluster')
        return all_props
    positions = all_props[['x', 'y']].copy()

    if all_props.shape[0] < 3:
        return all_props[['x', 'y', 'z', 'w', 'I']]

    # Hierarchical clustering
    dist_mat = squareform(pdist(positions.values))
    link_mat = hierarchy.linkage(dist_mat)
    cluster_idx = hierarchy.fcluster(link_mat, radius,
                                     criterion='distance')
    all_props['label'] = cluster_idx
    all_props.set_index('label', drop=False, append=False, inplace=True)
    all_props.index.name = 'label'
    all_props = all_props.sort_index()

    lbl_bincount = np.bincount(
        all_props.index.astype(np.uint16))[all_props.index.unique()]
    min_z_size = parameters['min_z_size']
    shorts = all_props.index.unique()[lbl_bincount < min_z_size]
    all_props = all_props.drop(shorts)
    all_props.set_index('label', drop=True, append=False, inplace=True)
    all_props.index.name = 'label'
    all_props = all_props.sort_index()
    return all_props

def get_cell_positions(all_props, interpolate=True):

    # Intensity should be summed, not averaged
    intensities = all_props['I'].groupby(
        level='label').sum()
    intensities /= intensities.max()
    cell_positions = all_props.groupby(
        level='label').apply(df_average, 'I')
    cell_positions['I'] = intensities
    return cell_positions

def df_average(df, weights_column):
    values = df.copy().iloc[0]
    norm = df[weights_column].sum()
    for col in df.columns:
        try:
            v = (df[col] * df[weights_column]).sum() / norm
        except TypeError:
            v = df[col].iloc[0]
        values[col] = v
    return values
<commit_msg>change in watershed argument<commit_after>
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

import logging
import itertools

import numpy as np
import pandas as pd

from scipy.spatial.distance import squareform, pdist
from scipy.cluster import hierarchy
from scipy import ndimage

from skimage.filter import rank, threshold_otsu
from skimage.morphology import disk, watershed
from skimage.feature import peak_local_max
from skimage.measure import regionprops

from ..io import ObjectsIO
from ..trajectories import Trajectories

log = logging.getLogger(__name__)

__all__ = []

DEFAULT_PARAMETERS = {'segment_method': 'otsu',
                      'correction': 1.,
                      'smooth': 10,
                      'min_radius': 2.,
                      'max_radius': 8.,
                      'num_cells': 8,
                      'nuc_distance':6,
                      'min_z_size': 4
                      }

def nuclei_detector(data_iterator,
                    metadata={},
                    parameters={},
                    parallel=False,
                    verbose=False,
                    mapper=None,
                    show_progress=False):
    '''
    TODO

    Parameters
    ----------
    data_iterator : Python iterator over dataset
        this iterator should yield one z-stack
        (corresponding to a single time point) at each
        iterations
    metadata: dict
        the metadata for the detection.
    parameters : dict
        the parameters for the detection
    parallel : bool
    verbose : bool
    mapper : bool
    show_progress : bool

    Returns
    -------
    trajs : :class:`Trajectories`
    '''

    _parameters = DEFAULT_PARAMETERS.copy()
    _parameters.update(parameters)
    parameters = _parameters

    parameters['min_z_size'] = parameters['min_z_size'] / metadata['PhysicalSizeZ']
    parameters['min_radius'] /= metadata['PhysicalSizeX']
    parameters['max_radius'] /= metadata['PhysicalSizeX']
    parameters['nuc_distance'] /= metadata['PhysicalSizeX']

    arguments = zip(data_iterator, itertools.repeat(parameters))
    if parallel and mapper is not None:
        results = mapper.map(detect_one_stack,
                             arguments)
    else:
        results = map(detect_one_stack,
                      arguments)

    raw_cell_positions = {}
    for i, result in enumerate(results):
        if 'positions' in result:
            raw_cell_positions[i] = result['positions']
    if not len(raw_cell_positions):
        log.warning('No cell detected anywhere')
        return Trajectories.empty_trajs(['x', 'y', 'z', 'w', 'I'])
    nuclei_positions = pd.concat(
        raw_cell_positions,
        keys=raw_cell_positions.keys(),
        names=('t_stamp', 'label'))

    real_times = nuclei_positions.index.get_level_values(
        't_stamp') * metadata['TimeIncrement']
    nuclei_positions['t'] = real_times
    nuclei_positions['t'] = nuclei_positions['t'].astype(np.float)
    nuclei_positions['x'] *=  metadata['PhysicalSizeX']
    nuclei_positions['y'] *=  metadata['PhysicalSizeY']
    nuclei_positions['z'] *=  metadata['PhysicalSizeZ']
    nuclei_positions['w'] *=  metadata['PhysicalSizeX']
    return Trajectories(nuclei_positions)

def detect_one_stack(args, full_output=False):

    z_stack, parameters = args
    if len(z_stack.shape) == 2:
        z_stack = z_stack[np.newaxis, ...]
    labeled_stack = label_stack(z_stack, parameters)
    if labeled_stack is None:
        return {}
    all_props = get_regionprops(labeled_stack,
                                z_stack, parameters)
    if not len(all_props):
        log.warning('No cell found after labelling')
        if full_output:
            output = {'labeled_stack': labeled_stack}
            return output
        else:
            return {}
    if z_stack.shape[0] == 1: ## Only one z plane
        positions = all_props.copy()
        positions.index = pd.Index(range(all_props.shape[0]))
        if full_output:
            output = {'positions': positions,
                      'labeled_stack': labeled_stack}
        else:
            output = {'positions': positions}
        return output

    agregated_props = cluster_regions(all_props, parameters)
    positions = get_cell_positions(agregated_props, parameters)

    if not len(positions):
        output = {'all_props': all_props,
                  'agregated_props': agregated_props,
                  'labeled_stack': labeled_stack}
        return output
    if full_output:
        output = {'positions': positions,
                  'all_props': all_props,
                  'agregated_props': agregated_props,
                  'labeled_stack': labeled_stack}
    else:
        output = {'positions': positions,}
    return output

def label_stack(z_stack, parameters):
    '''
    Apply an automated threshold and a watershed algorithm to
    label cells in each planes of the stack

    Parameters:
    -----------
    z_stack: 3d array of shape (nz, nx, ny)

    Returns:
    --------
    labeled_stack: 3d array of the same shape as z_stack,
    with each detected region labeled
    '''
    segment_method = parameters['segment_method']
    correction = parameters['correction']
    labeled_stack = np.zeros(z_stack.shape, dtype=np.uint8)
    max_int_proj = z_stack.max(axis=0)
    thresh = None

    if segment_method == 'otsu':
        thresh = threshold_otsu(max_int_proj) * correction
    elif segment_method == 'naive':
        thresh = max_int_proj.max() * correction
    else:
        err_string = ('Segmentation method {}'
                      'is not implemented'.format(segment_method))
        raise NotImplementedError(err_string)
    while thresh > z_stack.max():
        log.warning('''Reducing threshold''')
        thresh *= 0.9
    for n, frame in enumerate(z_stack):
        labeled_stack[n] = label_from_thresh(frame, thresh, parameters)
    return labeled_stack

def label_from_thresh(frame, thresh, parameters):

    smooth = parameters['smooth']
    min_distance = np.int(parameters['nuc_distance'])
    image = rank.median(frame, disk(smooth))
    image = rank.enhance_contrast(image, disk(smooth))
    im_max = image.max()
    if im_max < thresh:
        return np.zeros(image.shape, dtype=np.int32)
    else:
        image = image > thresh
        distance = ndimage.distance_transform_edt(image)
        local_maxi = peak_local_max(distance,
                                    footprint=disk(min_distance),
                                    #min_distance=min_distance,
                                    indices=False, labels=image)
        markers = ndimage.label(local_maxi)[0]
        return watershed(-distance, markers, mask=image)

def get_regionprops(labeled_stack, z_stack, parameters):
    '''
    Computes the area, centroid position and mean intensity
    of all the regions
    Removes the regions with typical size outside [min_radius, max_radius]

    returns a DataFrame with colmuns ('x', 'y', 'z', 'I', 'w')
    '''

    min_radius = parameters['min_radius']
    max_radius = parameters['max_radius']
    all_props = []
    #properties = ['Area', 'WeightedCentroid', 'MeanIntensity']
    columns = ('x', 'y', 'z', 'I', 'w')
    indices = []
    for z, frame in enumerate(labeled_stack):
        f_prop = regionprops(frame.astype(np.int),# properties,
                             intensity_image=z_stack[z])
        for d in f_prop:
            radius = (d.area / np.pi)**0.5
            if (min_radius  < radius < max_radius):
                all_props.append([d.centroid[0],
                                  d.centroid[1],
                                  z, d.mean_intensity * d.area,
                                  radius])
                indices.append((d.label, z))
    if not len(indices):
        return pd.DataFrame([], index=[])
    indices = pd.MultiIndex.from_tuples(indices, names=('label', 'z'))
    all_props = pd.DataFrame(all_props, index=indices, columns=columns)
    return all_props

def cluster_regions(all_props, parameters):

    radius = parameters['max_radius']
    n_clusters = all_props.shape[0]
    if not n_clusters:
        return None
    if n_clusters == 1:
        log.warning('Only one cluster')
        return all_props
    positions = all_props[['x', 'y']].copy()

    if all_props.shape[0] < 3:
        return all_props[['x', 'y', 'z', 'w', 'I']]

    # Hierarchical clustering
    dist_mat = squareform(pdist(positions.values))
    link_mat = hierarchy.linkage(dist_mat)
    cluster_idx = hierarchy.fcluster(link_mat, radius,
                                     criterion='distance')
    all_props['label'] = cluster_idx
    all_props.set_index('label', drop=False, append=False, inplace=True)
    all_props.index.name = 'label'
    all_props = all_props.sort_index()

    lbl_bincount = np.bincount(
        all_props.index.astype(np.uint16))[all_props.index.unique()]
    min_z_size = parameters['min_z_size']
    shorts = all_props.index.unique()[lbl_bincount < min_z_size]
    all_props = all_props.drop(shorts)
    all_props.set_index('label', drop=True, append=False, inplace=True)
    all_props.index.name = 'label'
    all_props = all_props.sort_index()
    return all_props

def get_cell_positions(all_props, interpolate=True):

    # Intensity should be summed, not averaged
    intensities = all_props['I'].groupby(
        level='label').sum()
    intensities /= intensities.max()
    cell_positions = all_props.groupby(
        level='label').apply(df_average, 'I')
    cell_positions['I'] = intensities
    return cell_positions

def df_average(df, weights_column):
    values = df.copy().iloc[0]
    norm = df[weights_column].sum()
    for col in df.columns:
        try:
            v = (df[col] * df[weights_column]).sum() / norm
        except TypeError:
            v = df[col].iloc[0]
        values[col] = v
    return values
<|endoftext|>


<commit_before>VERSION = (0, 1, 1)

def get_version():
    return '.'.join([str(bit) for bit in VERSION])
<commit_msg>Bump version<commit_after>VERSION = (0, 1, 2)

def get_version():
    return '.'.join([str(bit) for bit in VERSION])
<|endoftext|>


<commit_before>import argparse

"""
usage: mfh.py [-h] [-c | --client [PORT]] [-u] [-v]

Serve some sweet honey to the ubiquitous bots!

optional arguments:
  -h, --help       show this help message and exit
  -c               launch client with on port defined in settings
  --client [PORT]  port to start a client on
  -u, --updater    enable self updating
  -v, --verbose    increase output verbosity
"""

def parse():
    parser = argparse.ArgumentParser(
        description='Serve some sweet honey to the ubiquitous bots!',
        epilog='And that`s how you`d detect a sneaky chinese bot.',
        prog='mfh.py',
        )

    client_group = parser.add_mutually_exclusive_group()

    client_group.add_argument(
        '-c',
        action='store_true',
        help='launch client with on port defined in settings',
        )

    client_group.add_argument(
        '--client',
        help='port to start a client on',
        metavar='PORT',
        nargs='?',
        type=int,
        )

    server_group = parser.add_mutually_exclusive_group()

    server_group.add_argument(
        '-s',
        action='store_true',
        help='launch server with on port defined in settings',
        )

    server_group.add_argument(
        '--server',
        help='port to start a server on',
        metavar='PORT',
        nargs='?',
        type=int,
        )

    parser.add_argument(
        '-u',
        '--updater',
        action='store_true',
        help='enable self updating',
        )

    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
        help='increase output verbosity',
        )

    return parser.parse_args()
<commit_msg>Update docstring<commit_after>import argparse

"""
usage: mfh.py [-h] [-c | --client [PORT]] [-s | --server [PORT]] [-u] [-v]

Serve some sweet honey to the ubiquitous bots!

optional arguments:
  -h, --help       show this help message and exit
  -c               launch client with on port defined in settings
  --client [PORT]  port to start a client on
  -s               launch server with on port defined in settings
  --server [PORT]  port to start a server on
  -u, --updater    enable self updating
  -v, --verbose    increase output verbosity

And that`s how you`d detect a sneaky chinese bot.
"""

def parse():
    parser = argparse.ArgumentParser(
        description='Serve some sweet honey to the ubiquitous bots!',
        epilog='And that`s how you`d detect a sneaky chinese bot.',
        prog='mfh.py',
        )

    client_group = parser.add_mutually_exclusive_group()

    client_group.add_argument(
        '-c',
        action='store_true',
        help='launch client with on port defined in settings',
        )

    client_group.add_argument(
        '--client',
        help='port to start a client on',
        metavar='PORT',
        nargs='?',
        type=int,
        )

    server_group = parser.add_mutually_exclusive_group()

    server_group.add_argument(
        '-s',
        action='store_true',
        help='launch server with on port defined in settings',
        )

    server_group.add_argument(
        '--server',
        help='port to start a server on',
        metavar='PORT',
        nargs='?',
        type=int,
        )

    parser.add_argument(
        '-u',
        '--updater',
        action='store_true',
        help='enable self updating',
        )

    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
        help='increase output verbosity',
        )

    return parser.parse_args()
<|endoftext|>