<commit_before>import csv
from collections import defaultdict

from django.contrib.postgres.fields.jsonb import KeyTextTransform
from django.core.management.base import BaseCommand
from django.db.models import Count, IntegerField

from dimagi.utils.chunked import chunked
from django.db.models.functions import Cast

from corehq.apps.users.models import CouchUser
from corehq.util.argparse_types import date_type
from corehq.util.log import with_progress_bar
from custom.icds_reports.const import THR_REPORT_EXPORT
from custom.icds_reports.models import ICDSAuditEntryRecord
from custom.icds_reports.sqldata.exports.dashboard_usage import DashBoardUsage

prefix = 'dashboard_usage_data_'
TABULAR_DATA_CACHE = f'{prefix}tabular_data.csv'
CAS_DATA_CACHE = f'{prefix}cas_export_data.csv'

def _write_to_file(filename, rows):
    print(f'Writing {len(rows)} to file {filename}')
    with open(filename, 'w') as f:
        writer = csv.writer(f)
        writer.writerows([
            r if isinstance(r, list) else [r] for r in rows
        ])

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            'start_date',
            type=date_type,
            help='The start date (inclusive). format YYYY-MM-DD'
        )
        parser.add_argument(
            'end_date',
            type=date_type,
            help='The end date (exclusive). format YYYY-MM-DD'
        )
        parser.add_argument('domain')
        parser.add_argument('username', help='The username of the logged in user')

    def handle(self, start_date, end_date, domain, username, **options):
        print(f'fetching users data')

        user = CouchUser.get_by_username(username)
        usage_data = DashBoardUsage(couch_user=user, domain=domain).get_excel_data()[0][1]

        # fetching username to state mapping
        username_vs_state_name = defaultdict()
        for data in usage_data[1:]:
            username = data[4]
            state_name = data[1]
            username_vs_state_name[username] = state_name

        self.get_tabular_data(usage_data, start_date, end_date, domain)
        print(f'Request data written to file {TABULAR_DATA_CACHE}')
        self.get_cas_data(username_vs_state_name, start_date, end_date, domain)
        print(f'Request data written to file {CAS_DATA_CACHE}')

    def get_dashboard_tabular_usage_counts(self, start_date, end_date, domain):
        """
        :param start_date: start date of the filter
        :param end_date: end date of the filter
        :param domain
        :return: returns the counts of no of downloads of each and total reports for  all usernames
        """
        print(f'Compiling tabular usage counts for users')
        tabular_user_counts = defaultdict(int)
        tabular_user_indicators = defaultdict(lambda: [0] * 10)

        records = list(ICDSAuditEntryRecord.objects.filter(url=f'/a/{domain}/icds_export_indicator',
                                                           time_of_use__gte=start_date,
                                                           time_of_use__lte=end_date)
                       .annotate(indicator=Cast(KeyTextTransform('indicator', 'post_data'), IntegerField()))
                       .filter(indicator__lte=THR_REPORT_EXPORT).values('indicator', 'username')
                       .annotate(count=Count('indicator')).order_by('username', 'indicator'))
        for record in records:
            tabular_user_counts[record['username'].split('@')[0]] += record['count']
            tabular_user_indicators[record['username'].split('@')[0]][int(record['indicator']) - 1]\
                = record['count']

        return tabular_user_counts, tabular_user_indicators

    def get_dashboard_cas_usage_counts(self, start_date, end_date, domain):
        """
        :param start_date: start date of the filter
        :param end_date: end date of the filter
        :param domain
        :return: returns the counts of no of downloads of each and total reports for  all usernames
        """
        print(f'Compiling cas export usage counts for users')
        cas_user_counts = defaultdict(int)

        records = list(ICDSAuditEntryRecord.objects.filter(url=f'/a/{domain}/cas_export',
                                                           time_of_use__gte=start_date,
                                                           time_of_use__lte=end_date)
                       .annotate(indicator=Cast(KeyTextTransform('indicator', 'post_data'), IntegerField()))
                       .values('indicator')
                       .annotate(count=Count('indicator')).values('username', 'count').order_by('username'))
        for record in records:
            cas_user_counts[record['username'].split('@')[0]] += record['count']
        return cas_user_counts

    def get_tabular_data(self, usage_data, start_date, end_date, domain):
        tab_data = list()

        tab_usage_counts, tab_indicators_count = \
            self.get_dashboard_tabular_usage_counts(start_date, end_date, domain)

        extra_headers = ['Total', 'Child', 'Pregnant Women', 'Demographics', 'System Usage',
                         'AWC infrastructure', 'Child Growth Monitoring List', 'ICDS - CAS Monthly Register',
                         'AWW Performance Report', 'LS Performance Report', 'Take Home Ration']
        print(f'Compiling request data for {len(usage_data)} users')
        headers = usage_data[0][:7]
        headers.extend(extra_headers)
        usage_data = usage_data[1:]
        for chunk in chunked(with_progress_bar(usage_data), 500):
            for data in chunk:
                username = data[4]
                indicator_count = tab_indicators_count[username]
                csv_row = data[:7]
                csv_row.append(tab_usage_counts[username])
                csv_row.extend(indicator_count)
                tab_data.append(csv_row)

        tab_data.insert(0, headers)
        _write_to_file(TABULAR_DATA_CACHE, tab_data)

    def get_cas_data(self, username_state_mapping, start_date, end_date, domain):
        cas_total_counts = self.get_dashboard_cas_usage_counts(start_date, end_date, domain)
        sheet_headers = ['Sr.No', 'State/UT Name',
                         'No. of times CAS data export downloaded (December 2019)']
        cas_data = list()
        cas_data_dict = defaultdict(int)

        # converting usernames to state names
        for key, value in cas_total_counts.items():
            cas_data_dict[username_state_mapping[key]] += value

        # creating cas data
        serial = 0
        for key, value in cas_data_dict.items():
            serial += 1
            cas_data.append([serial, key, value])
        cas_data.insert(0, sheet_headers)
        _write_to_file(CAS_DATA_CACHE, cas_data)
<commit_msg>changed the cast value from integer to text<commit_after>import csv
from collections import defaultdict

from django.contrib.postgres.fields.jsonb import KeyTextTransform
from django.core.management.base import BaseCommand
from django.db.models import Count, IntegerField, TextField

from dimagi.utils.chunked import chunked
from django.db.models.functions import Cast

from corehq.apps.users.models import CouchUser
from corehq.util.argparse_types import date_type
from corehq.util.log import with_progress_bar
from custom.icds_reports.const import THR_REPORT_EXPORT
from custom.icds_reports.models import ICDSAuditEntryRecord
from custom.icds_reports.sqldata.exports.dashboard_usage import DashBoardUsage

prefix = 'dashboard_usage_data_'
TABULAR_DATA_CACHE = f'{prefix}tabular_data.csv'
CAS_DATA_CACHE = f'{prefix}cas_export_data.csv'

def _write_to_file(filename, rows):
    print(f'Writing {len(rows)} to file {filename}')
    with open(filename, 'w') as f:
        writer = csv.writer(f)
        writer.writerows([
            r if isinstance(r, list) else [r] for r in rows
        ])

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            'start_date',
            type=date_type,
            help='The start date (inclusive). format YYYY-MM-DD'
        )
        parser.add_argument(
            'end_date',
            type=date_type,
            help='The end date (exclusive). format YYYY-MM-DD'
        )
        parser.add_argument('domain')
        parser.add_argument('username', help='The username of the logged in user')

    def handle(self, start_date, end_date, domain, username, **options):
        print(f'fetching users data')

        user = CouchUser.get_by_username(username)
        usage_data = DashBoardUsage(couch_user=user, domain=domain).get_excel_data()[0][1]

        # fetching username to state mapping
        username_vs_state_name = defaultdict()
        for data in usage_data[1:]:
            username = data[4]
            state_name = data[1]
            username_vs_state_name[username] = state_name

        self.get_tabular_data(usage_data, start_date, end_date, domain)
        print(f'Request data written to file {TABULAR_DATA_CACHE}')
        self.get_cas_data(username_vs_state_name, start_date, end_date, domain)
        print(f'Request data written to file {CAS_DATA_CACHE}')

    def get_dashboard_tabular_usage_counts(self, start_date, end_date, domain):
        """
        :param start_date: start date of the filter
        :param end_date: end date of the filter
        :param domain
        :return: returns the counts of no of downloads of each and total reports for  all usernames
        """
        print(f'Compiling tabular usage counts for users')
        tabular_user_counts = defaultdict(int)
        tabular_user_indicators = defaultdict(lambda: [0] * 10)

        records = list(ICDSAuditEntryRecord.objects.filter(url=f'/a/{domain}/icds_export_indicator',
                                                           time_of_use__gte=start_date,
                                                           time_of_use__lte=end_date)
                       .annotate(indicator=Cast(KeyTextTransform('indicator', 'post_data'), IntegerField()))
                       .filter(indicator__lte=THR_REPORT_EXPORT).values('indicator', 'username')
                       .annotate(count=Count('indicator')).order_by('username', 'indicator'))
        for record in records:
            tabular_user_counts[record['username'].split('@')[0]] += record['count']
            tabular_user_indicators[record['username'].split('@')[0]][int(record['indicator']) - 1]\
                = record['count']

        return tabular_user_counts, tabular_user_indicators

    def get_dashboard_cas_usage_counts(self, start_date, end_date, domain):
        """
        :param start_date: start date of the filter
        :param end_date: end date of the filter
        :param domain
        :return: returns the counts of no of downloads of each and total reports for  all usernames
        """
        print(f'Compiling cas export usage counts for users')
        cas_user_counts = defaultdict(int)

        records = list(ICDSAuditEntryRecord.objects.filter(url=f'/a/{domain}/cas_export',
                                                           time_of_use__gte=start_date,
                                                           time_of_use__lte=end_date)
                       .annotate(indicator=Cast(KeyTextTransform('indicator', 'post_data'), TextField()))
                       .values('indicator')
                       .annotate(count=Count('indicator')).values('username', 'count').order_by('username'))
        for record in records:
            cas_user_counts[record['username'].split('@')[0]] += record['count']
        return cas_user_counts

    def get_tabular_data(self, usage_data, start_date, end_date, domain):
        tab_data = list()

        tab_usage_counts, tab_indicators_count = \
            self.get_dashboard_tabular_usage_counts(start_date, end_date, domain)

        extra_headers = ['Total', 'Child', 'Pregnant Women', 'Demographics', 'System Usage',
                         'AWC infrastructure', 'Child Growth Monitoring List', 'ICDS - CAS Monthly Register',
                         'AWW Performance Report', 'LS Performance Report', 'Take Home Ration']
        print(f'Compiling request data for {len(usage_data)} users')
        headers = usage_data[0][:7]
        headers.extend(extra_headers)
        usage_data = usage_data[1:]
        for chunk in chunked(with_progress_bar(usage_data), 500):
            for data in chunk:
                username = data[4]
                indicator_count = tab_indicators_count[username]
                csv_row = data[:7]
                csv_row.append(tab_usage_counts[username])
                csv_row.extend(indicator_count)
                tab_data.append(csv_row)

        tab_data.insert(0, headers)
        _write_to_file(TABULAR_DATA_CACHE, tab_data)

    def get_cas_data(self, username_state_mapping, start_date, end_date, domain):
        cas_total_counts = self.get_dashboard_cas_usage_counts(start_date, end_date, domain)
        sheet_headers = ['Sr.No', 'State/UT Name',
                         'No. of times CAS data export downloaded (December 2019)']
        cas_data = list()
        cas_data_dict = defaultdict(int)

        # converting usernames to state names
        for key, value in cas_total_counts.items():
            cas_data_dict[username_state_mapping[key]] += value

        # creating cas data
        serial = 0
        for key, value in cas_data_dict.items():
            serial += 1
            cas_data.append([serial, key, value])
        cas_data.insert(0, sheet_headers)
        _write_to_file(CAS_DATA_CACHE, cas_data)
<|endoftext|>


<commit_before>#! /usr/bin/python
import roslib; roslib.load_manifest('ptp_arm_action')
import rospy
import actionlib

import geometry_msgs.msg as gm
import ptp_arm_action.msg as ptp

from object_manipulator.convert_functions import *
from math import sqrt, pi, fabs
import tf_utils as tfu
import numpy as np
#import scipy
import math
import tf.transformations as tr
import tf
from pycontroller_manager.pycontroller_manager import ControllerManager
import object_manipulator.convert_functions as cf

##
# Measures the distance between two pose stamps, independently factors out
# distance in rotation and distance in translation. Converts both poses into
# the same coordinate frame before comparison.
def pose_distance(ps_a, ps_b, tflistener):
    #put both into the same frame
    ps_a_fb = change_pose_stamped_frame(tflistener, ps_a, ps_b.header.frame_id)
    g_T_a = pose_to_mat(ps_a_fb.pose)
    g_T_b = pose_to_mat(ps_b.pose)

    a_T_b = g_T_a**-1 * g_T_b

    desired_trans = a_T_b[0:3, 3].copy()
    a_T_b[0:3, 3] = 0
    desired_angle, desired_axis, _ = tf.transformations.rotation_from_matrix(a_T_b)
    return desired_trans, desired_angle, desired_axis

def tf_as_matrix(tup):
    return np.matrix(tr.translation_matrix(tup[0])) * np.matrix(tr.quaternion_matrix(tup[1]))

class PTPArmActionServer:

    def __init__(self, name, arm):
        if arm == 'left':
            self.controller = 'l_cart'
            #self.joint_controller = 'l_arm_controller'
            self.controller_frame = rospy.get_param('/l_cart/tip_name')
            #self.tool_frame = rospy.get_param('/l_cart/tip_name')
            self.tool_frame = 'l_gripper_tool_frame'
        elif arm == 'right':
            self.controller = 'r_cart'
            #self.joint_controller = 'r_arm_controller'
            self.controller_frame = rospy.get_param('/r_cart/tip_name')
            #self.tool_frame = rospy.get_param('/r_cart/tip_name')
            self.tool_frame = 'r_gripper_tool_frame'
        else:
            raise RuntimeError('Invalid parameter for arm: %s' % arm)

        self.arm = arm
        self.target_pub = rospy.Publisher(self.controller + '/command_pose', gm.PoseStamped)
        #self.pose_sub = rospy.Subscriber(self.controller + '/state/x', gm.PoseStamped, self.pose_callback)

        self.tf = tf.TransformListener()
        #self.trans_tolerance = rospy.get_param("~translation_tolerance")
        self.rot_tolerance = math.radians(rospy.get_param("~rotation_tolerance"))
        self.stall_time = rospy.get_param("~stall_time")
        #rospy.loginfo('trans tolerance ' + str(self.trans_tolerance))
        #self.time_out = rospy.get_param("~timeout")

        self.controller_manager = ControllerManager()
        #self.controller_manager.cart_mode(self.arm)
        self._action_name = name
        self.linear_movement_as = actionlib.SimpleActionServer(self._action_name, ptp.LinearMovementAction,
					execute_cb=self.action_cb, auto_start=False)
        self.linear_movement_as.start()

        rospy.loginfo('Action name: %s Arm: %s' % (self._action_name, self.arm))

    def action_cb(self, msg):
        rospy.loginfo('message that we got:\n' + str(msg))
        self.controller_manager.cart_mode(self.arm)
        trans_tolerance = msg.trans_tolerance #rospy.get_param("~translation_tolerance")
        time_out = msg.time_out #rospy.get_param("~timeout")
        self.rot_tolerance = math.radians(rospy.get_param("~rotation_tolerance"))

        #self._wait_for_pose_message()

        success = False
        r = rospy.Rate(100)

        goal_ps = msg.goal
        #relative_movement = msg.relative
        trans_vel = msg.trans_vel
        rot_vel = msg.rot_vel
        if trans_vel <= 0:
            trans_vel = .02
        if rot_vel <= 0:
            rot_vel = pi/20.

        tstart = rospy.get_time()
        tmax = tstart + time_out
        #self.controller_manager = ControllerManager()
        rospy.loginfo('Goal is x %f y %f z %f in %s' % (goal_ps.pose.position.x, goal_ps.pose.position.y,
            goal_ps.pose.position.z, goal_ps.header.frame_id))

        goal_torso = change_pose_stamped_frame(self.tf, goal_ps, 'torso_lift_link')
        rospy.loginfo('BEFORE TIP Goal is x %f y %f z %f in %s' % (goal_torso.pose.position.x, goal_torso.pose.position.y,
            goal_torso.pose.position.z, goal_torso.header.frame_id))

        rospy.loginfo('Tool Frame %s Controller Frame %s' % (self.tool_frame, self.controller_frame))

        tll_T_tip = pose_to_mat(goal_torso.pose)
        tip_T_w = tf_as_matrix(self.tf.lookupTransform(self.tool_frame, self.controller_frame,  rospy.Time(0)))
        #print 'TRANSFORM', tip_T_w
        tll_T_w = tll_T_tip * tip_T_w
        goal_torso = stamp_pose(mat_to_pose(tll_T_w), 'torso_lift_link')

        rospy.loginfo('AFTER TIP Goal is x %f y %f z %f in %s' % (goal_torso.pose.position.x, goal_torso.pose.position.y,
            goal_torso.pose.position.z, goal_torso.header.frame_id))

        verbose = False
        time_ang = None
        min_ang_error = None
        time_trans = None
        min_trans_error = None
        timed_out = False

        while True:
            cur_time = rospy.get_time()

            gripper_matrix = tfu.tf_as_matrix(self.tf.lookupTransform('torso_lift_link', self.controller_frame, rospy.Time(0)))
            gripper_ps = stamp_pose(mat_to_pose(gripper_matrix), 'torso_lift_link')
            #Someone preempted us!
            if self.linear_movement_as.is_preempt_requested():
                #Stop our motion
                self.target_pub.publish(stamp_pose(gripper_ps.pose, gripper_ps.header.frame_id))
                self.linear_movement_as.set_preempted()
                rospy.loginfo('action_cb: preempted!')
                break

            #Calc feedback
            if verbose:
                print 'current_pose %.3f %.3f %.3f, rot %.3f %.3f %.3f %.3f in %s' % (gripper_ps.pose.position.x, gripper_ps.pose.position.y, gripper_ps.pose.position.z,
                        gripper_ps.pose.orientation.x, gripper_ps.pose.orientation.y, gripper_ps.pose.orientation.z,
                        gripper_ps.pose.orientation.w, gripper_ps.header.frame_id)
                print 'goal_pose %.3f %.3f %.3f, rot %.3f %.3f %.3f %.3f in %s' % (goal_torso.pose.position.x, goal_torso.pose.position.y, goal_torso.pose.position.z,
                        goal_torso.pose.orientation.x, goal_torso.pose.orientation.y, goal_torso.pose.orientation.z,
                        goal_torso.pose.orientation.w, goal_torso.header.frame_id)

            trans, ang, _ = pose_distance(gripper_ps, goal_torso, self.tf)
            feedback = ptp.LinearMovementFeedback(gm.Vector3(trans[0,0], trans[1,0], trans[2,0]))
            self.linear_movement_as.publish_feedback(feedback)

            #Reached goal
            trans_mag = np.linalg.norm(trans)
            if verbose:
                print trans.T, 'trans_mag', trans_mag, 'ang', abs(ang), 'rot toler', self.rot_tolerance

            if min_trans_error == None or min_trans_error == None:
                min_trans_error = trans_mag
                min_ang_error = abs(ang)
                time_ang = cur_time
                time_trans = cur_time

            if trans_mag < min_trans_error:
                min_trans_error = trans_mag
                time_trans = cur_time

            if abs(ang) < min_ang_error:
                min_ang_error = abs(ang)
                time_ang = cur_time

            if trans_tolerance > trans_mag and self.rot_tolerance > abs(ang):
                rospy.loginfo('action_cb: reached goal.')
                break

            #Timed out! is this a failure?
            if cur_time > tmax:
                rospy.loginfo('action_cb: timed out.')
                timed_out = True
                break

            #if it has been a while since we made progress
            if trans_mag > min_trans_error and (cur_time - time_trans) > self.stall_time:
                rospy.loginfo('action_cb: stalled.')
                break

            # tends to not stall on angle so don't look out for this case
            #if abs(ang) > min_ang_error and (cur_time - time_ang) > self.stall_time:
            #    rospy.loginfo('action_cb: stalled.')
            #    break

            #Send controls
            clamped_target = self.clamp_pose(goal_torso, trans_vel, rot_vel, ref_pose=gripper_ps)
            if verbose:
                print 'clamped_target', clamped_target.pose.position.x, clamped_target.pose.position.y,
                print clamped_target.pose.position.z, clamped_target.header.frame_id, '\n'

            #break
            self.target_pub.publish(clamped_target)
            #break

        trans, ang, _ = pose_distance(gripper_ps, goal_torso, self.tf)
        result = ptp.LinearMovementResult(gm.Vector3(trans[0,0], trans[1,0], trans[2,0]), 'unknown')
        if trans_tolerance > np.linalg.norm(trans):
            rospy.loginfo( 'SUCCEEDED! %.3f ang %.3f' % (np.linalg.norm(trans), np.degrees(ang)))
            result.message = 'succeeded'
            self.linear_movement_as.set_succeeded(result)
        else:
            rospy.loginfo('ABORTED! %.3f ang %.3f' % (np.linalg.norm(trans), np.degrees(ang)))
            if timed_out:
                result.message = 'timed_out'
            else:
                result.message = 'goal_not_reached'
            self.linear_movement_as.set_aborted(result)

    def clamp_pose(self, desired_pose, max_trans, max_rot, ref_pose):
        current_pose_d = change_pose_stamped_frame(self.tf, ref_pose, desired_pose.header.frame_id)
        g_T_c  = pose_to_mat(current_pose_d.pose)

        desired_trans, desired_angle, desired_axis = pose_distance(ref_pose, desired_pose, self.tf)
        desired_trans_mag = np.linalg.norm(desired_trans)
        frac_trans = fabs(desired_trans_mag / max_trans)
        frac_rot = fabs(desired_angle / max_rot)

        if frac_trans <= 1 and frac_rot <= 1:
            return desired_pose
        frac = max(frac_rot, frac_trans)

        clamped_angle = desired_angle / frac
        clamped_trans = desired_trans / frac

        c_T_d = np.matrix(tf.transformations.rotation_matrix(clamped_angle, desired_axis))
        c_T_d[0:3, 3] = clamped_trans
        g_T_d = g_T_c * c_T_d
        clamped_pose = stamp_pose(mat_to_pose(g_T_d), desired_pose.header.frame_id)
        clamped_pose.header.stamp = rospy.Time.now()
        return clamped_pose

if __name__ == '__main__':
    import sys

    if len(sys.argv) < 2:
        arm = 'left'
    else:
        arm = sys.argv[1]

    rospy.init_node('ptp')
    left_as = PTPArmActionServer(arm +'_ptp', arm)
    rospy.loginfo('PTP action server started.')
    rospy.spin()

        #if relative_movement:
        #    rospy.loginfo('Received relative motion.')

        #    #print 'tool frame is', self.tool_frame
        #    #print 'goal frame is', goal_ps.header.frame_id

        #    delta_ref  = pose_to_mat(goal_ps.pose)
        #    tll_R_ref = tfu.tf_as_matrix(self.tf.lookupTransform('torso_lift_link', goal_ps.header.frame_id, rospy.Time(0)))
        #    tll_R_ref[0:3,3] = 0
        #    delta_tll = tll_R_ref * delta_ref

        #    #print 'delta_tll\n', delta_tll
        #    tip_current_T_tll = tfu.tf_as_matrix(self.tf.lookupTransform('torso_lift_link', self.tool_frame, rospy.Time(0)))
        #    #print 'tip_current_T_tll\n', tip_current_T_tll

        #    #Find translation
        #    delta_T = delta_tll.copy()
        #    delta_T[0:3,0:3] = np.eye(3)
        #    tip_T = delta_T * tip_current_T_tll

        #    #Find rotation
        #    tip_R = delta_tll[0:3, 0:3] * tip_current_T_tll[0:3, 0:3]

        #    tip_new = np.matrix(np.eye(4))
        #    tip_new[0:3, 0:3] = tip_R
        #    tip_new[0:3, 3] = tip_T[0:3,3]

        #    #print 'tip_new\n', tip_new
        #    goal_ps = stamp_pose(mat_to_pose(tip_new), 'torso_lift_link')

<commit_msg>removed dep on object manipulator functions<commit_after>#! /usr/bin/python
import roslib; roslib.load_manifest('ptp_arm_action')
import rospy
import actionlib

import geometry_msgs.msg as gm
import ptp_arm_action.msg as ptp

#from object_manipulator.convert_functions import *
from math import sqrt, pi, fabs
import tf_utils as tfu
import numpy as np
#import scipy
import math
import tf.transformations as tr
import tf
from pycontroller_manager.pycontroller_manager import ControllerManager
import object_manipulator.convert_functions as cf

##make a PoseStamped out of a Pose
def stamp_pose(pose, frame_id):
    pose_stamped = PoseStamped()
    stamp_msg(pose_stamped, frame_id)
    pose_stamped.pose = pose
    return pose_stamped

##change the frame of a PoseStamped
def change_pose_stamped_frame(tf_listener, pose, frame):

    #convert the PoseStamped to the desired frame, if necessary
    if pose.header.frame_id != frame:
        pose.header.stamp = rospy.Time(0)
        tf_listener.waitForTransform(frame, pose.header.frame_id, pose.header.stamp, rospy.Duration(5))
        try:
            trans_pose = tf_listener.transformPose(frame, pose)
        except rospy.ServiceException, e:
            print "pose:\n", pose
            print "frame:", frame
            rospy.logerr("change_pose_stamped_frame: error in transforming pose from " + pose.header.frame_id + " to " + frame + "error msg: %s"%e)
            return None
    else:
        trans_pose = pose

    return trans_pose

def pose_to_mat(pose):
    quat = [pose.orientation.x, pose.orientation.y, pose.orientation.z, pose.orientation.w]
    pos = np.matrix([pose.position.x, pose.position.y, pose.position.z]).T
    mat = np.matrix(tf.transformations.quaternion_matrix(quat))
    mat[0:3, 3] = pos
    return mat

##
# Measures the distance between two pose stamps, independently factors out
# distance in rotation and distance in translation. Converts both poses into
# the same coordinate frame before comparison.
def pose_distance(ps_a, ps_b, tflistener):
    #put both into the same frame
    ps_a_fb = change_pose_stamped_frame(tflistener, ps_a, ps_b.header.frame_id)
    g_T_a = pose_to_mat(ps_a_fb.pose)
    g_T_b = pose_to_mat(ps_b.pose)

    a_T_b = g_T_a**-1 * g_T_b

    desired_trans = a_T_b[0:3, 3].copy()
    a_T_b[0:3, 3] = 0
    desired_angle, desired_axis, _ = tf.transformations.rotation_from_matrix(a_T_b)
    return desired_trans, desired_angle, desired_axis

def tf_as_matrix(tup):
    return np.matrix(tr.translation_matrix(tup[0])) * np.matrix(tr.quaternion_matrix(tup[1]))

class PTPArmActionServer:

    def __init__(self, name, arm):
        if arm == 'left':
            self.controller = 'l_cart'
            #self.joint_controller = 'l_arm_controller'
            self.controller_frame = rospy.get_param('/l_cart/tip_name')
            #self.tool_frame = rospy.get_param('/l_cart/tip_name')
            self.tool_frame = 'l_gripper_tool_frame'
        elif arm == 'right':
            self.controller = 'r_cart'
            #self.joint_controller = 'r_arm_controller'
            self.controller_frame = rospy.get_param('/r_cart/tip_name')
            #self.tool_frame = rospy.get_param('/r_cart/tip_name')
            self.tool_frame = 'r_gripper_tool_frame'
        else:
            raise RuntimeError('Invalid parameter for arm: %s' % arm)

        self.arm = arm
        self.target_pub = rospy.Publisher(self.controller + '/command_pose', gm.PoseStamped)
        #self.pose_sub = rospy.Subscriber(self.controller + '/state/x', gm.PoseStamped, self.pose_callback)

        self.tf = tf.TransformListener()
        #self.trans_tolerance = rospy.get_param("~translation_tolerance")
        self.rot_tolerance = math.radians(rospy.get_param("~rotation_tolerance"))
        self.stall_time = rospy.get_param("~stall_time")
        #rospy.loginfo('trans tolerance ' + str(self.trans_tolerance))
        #self.time_out = rospy.get_param("~timeout")

        self.controller_manager = ControllerManager()
        #self.controller_manager.cart_mode(self.arm)
        self._action_name = name
        self.linear_movement_as = actionlib.SimpleActionServer(self._action_name, ptp.LinearMovementAction,
					execute_cb=self.action_cb, auto_start=False)
        self.linear_movement_as.start()

        rospy.loginfo('Action name: %s Arm: %s' % (self._action_name, self.arm))

    def action_cb(self, msg):
        rospy.loginfo('message that we got:\n' + str(msg))
        self.controller_manager.cart_mode(self.arm)
        trans_tolerance = msg.trans_tolerance #rospy.get_param("~translation_tolerance")
        time_out = msg.time_out #rospy.get_param("~timeout")
        self.rot_tolerance = math.radians(rospy.get_param("~rotation_tolerance"))

        #self._wait_for_pose_message()

        success = False
        r = rospy.Rate(100)

        goal_ps = msg.goal
        #relative_movement = msg.relative
        trans_vel = msg.trans_vel
        rot_vel = msg.rot_vel
        if trans_vel <= 0:
            trans_vel = .02
        if rot_vel <= 0:
            rot_vel = pi/20.

        tstart = rospy.get_time()
        tmax = tstart + time_out
        #self.controller_manager = ControllerManager()
        rospy.loginfo('Goal is x %f y %f z %f in %s' % (goal_ps.pose.position.x, goal_ps.pose.position.y,
            goal_ps.pose.position.z, goal_ps.header.frame_id))

        goal_torso = change_pose_stamped_frame(self.tf, goal_ps, 'torso_lift_link')
        rospy.loginfo('BEFORE TIP Goal is x %f y %f z %f in %s' % (goal_torso.pose.position.x, goal_torso.pose.position.y,
            goal_torso.pose.position.z, goal_torso.header.frame_id))

        rospy.loginfo('Tool Frame %s Controller Frame %s' % (self.tool_frame, self.controller_frame))

        tll_T_tip = pose_to_mat(goal_torso.pose)
        tip_T_w = tf_as_matrix(self.tf.lookupTransform(self.tool_frame, self.controller_frame,  rospy.Time(0)))
        #print 'TRANSFORM', tip_T_w
        tll_T_w = tll_T_tip * tip_T_w
        goal_torso = stamp_pose(mat_to_pose(tll_T_w), 'torso_lift_link')

        rospy.loginfo('AFTER TIP Goal is x %f y %f z %f in %s' % (goal_torso.pose.position.x, goal_torso.pose.position.y,
            goal_torso.pose.position.z, goal_torso.header.frame_id))

        verbose = False
        time_ang = None
        min_ang_error = None
        time_trans = None
        min_trans_error = None
        timed_out = False

        while True:
            cur_time = rospy.get_time()

            gripper_matrix = tfu.tf_as_matrix(self.tf.lookupTransform('torso_lift_link', self.controller_frame, rospy.Time(0)))
            gripper_ps = stamp_pose(mat_to_pose(gripper_matrix), 'torso_lift_link')
            #Someone preempted us!
            if self.linear_movement_as.is_preempt_requested():
                #Stop our motion
                self.target_pub.publish(stamp_pose(gripper_ps.pose, gripper_ps.header.frame_id))
                self.linear_movement_as.set_preempted()
                rospy.loginfo('action_cb: preempted!')
                break

            #Calc feedback
            if verbose:
                print 'current_pose %.3f %.3f %.3f, rot %.3f %.3f %.3f %.3f in %s' % (gripper_ps.pose.position.x, gripper_ps.pose.position.y, gripper_ps.pose.position.z,
                        gripper_ps.pose.orientation.x, gripper_ps.pose.orientation.y, gripper_ps.pose.orientation.z,
                        gripper_ps.pose.orientation.w, gripper_ps.header.frame_id)
                print 'goal_pose %.3f %.3f %.3f, rot %.3f %.3f %.3f %.3f in %s' % (goal_torso.pose.position.x, goal_torso.pose.position.y, goal_torso.pose.position.z,
                        goal_torso.pose.orientation.x, goal_torso.pose.orientation.y, goal_torso.pose.orientation.z,
                        goal_torso.pose.orientation.w, goal_torso.header.frame_id)

            trans, ang, _ = pose_distance(gripper_ps, goal_torso, self.tf)
            feedback = ptp.LinearMovementFeedback(gm.Vector3(trans[0,0], trans[1,0], trans[2,0]))
            self.linear_movement_as.publish_feedback(feedback)

            #Reached goal
            trans_mag = np.linalg.norm(trans)
            if verbose:
                print trans.T, 'trans_mag', trans_mag, 'ang', abs(ang), 'rot toler', self.rot_tolerance

            if min_trans_error == None or min_trans_error == None:
                min_trans_error = trans_mag
                min_ang_error = abs(ang)
                time_ang = cur_time
                time_trans = cur_time

            if trans_mag < min_trans_error:
                min_trans_error = trans_mag
                time_trans = cur_time

            if abs(ang) < min_ang_error:
                min_ang_error = abs(ang)
                time_ang = cur_time

            if trans_tolerance > trans_mag and self.rot_tolerance > abs(ang):
                rospy.loginfo('action_cb: reached goal.')
                break

            #Timed out! is this a failure?
            if cur_time > tmax:
                rospy.loginfo('action_cb: timed out.')
                timed_out = True
                break

            #if it has been a while since we made progress
            if trans_mag > min_trans_error and (cur_time - time_trans) > self.stall_time:
                rospy.loginfo('action_cb: stalled.')
                break

            # tends to not stall on angle so don't look out for this case
            #if abs(ang) > min_ang_error and (cur_time - time_ang) > self.stall_time:
            #    rospy.loginfo('action_cb: stalled.')
            #    break

            #Send controls
            clamped_target = self.clamp_pose(goal_torso, trans_vel, rot_vel, ref_pose=gripper_ps)
            if verbose:
                print 'clamped_target', clamped_target.pose.position.x, clamped_target.pose.position.y,
                print clamped_target.pose.position.z, clamped_target.header.frame_id, '\n'

            #break
            self.target_pub.publish(clamped_target)
            #break

        trans, ang, _ = pose_distance(gripper_ps, goal_torso, self.tf)
        result = ptp.LinearMovementResult(gm.Vector3(trans[0,0], trans[1,0], trans[2,0]), 'unknown')
        if trans_tolerance > np.linalg.norm(trans):
            rospy.loginfo( 'SUCCEEDED! %.3f ang %.3f' % (np.linalg.norm(trans), np.degrees(ang)))
            result.message = 'succeeded'
            self.linear_movement_as.set_succeeded(result)
        else:
            rospy.loginfo('ABORTED! %.3f ang %.3f' % (np.linalg.norm(trans), np.degrees(ang)))
            if timed_out:
                result.message = 'timed_out'
            else:
                result.message = 'goal_not_reached'
            self.linear_movement_as.set_aborted(result)

    def clamp_pose(self, desired_pose, max_trans, max_rot, ref_pose):
        current_pose_d = change_pose_stamped_frame(self.tf, ref_pose, desired_pose.header.frame_id)
        g_T_c  = pose_to_mat(current_pose_d.pose)

        desired_trans, desired_angle, desired_axis = pose_distance(ref_pose, desired_pose, self.tf)
        desired_trans_mag = np.linalg.norm(desired_trans)
        frac_trans = fabs(desired_trans_mag / max_trans)
        frac_rot = fabs(desired_angle / max_rot)

        if frac_trans <= 1 and frac_rot <= 1:
            return desired_pose
        frac = max(frac_rot, frac_trans)

        clamped_angle = desired_angle / frac
        clamped_trans = desired_trans / frac

        c_T_d = np.matrix(tf.transformations.rotation_matrix(clamped_angle, desired_axis))
        c_T_d[0:3, 3] = clamped_trans
        g_T_d = g_T_c * c_T_d
        clamped_pose = stamp_pose(mat_to_pose(g_T_d), desired_pose.header.frame_id)
        clamped_pose.header.stamp = rospy.Time.now()
        return clamped_pose

if __name__ == '__main__':
    import sys

    if len(sys.argv) < 2:
        arm = 'left'
    else:
        arm = sys.argv[1]

    rospy.init_node('ptp')
    left_as = PTPArmActionServer(arm +'_ptp', arm)
    rospy.loginfo('PTP action server started.')
    rospy.spin()

        #if relative_movement:
        #    rospy.loginfo('Received relative motion.')

        #    #print 'tool frame is', self.tool_frame
        #    #print 'goal frame is', goal_ps.header.frame_id

        #    delta_ref  = pose_to_mat(goal_ps.pose)
        #    tll_R_ref = tfu.tf_as_matrix(self.tf.lookupTransform('torso_lift_link', goal_ps.header.frame_id, rospy.Time(0)))
        #    tll_R_ref[0:3,3] = 0
        #    delta_tll = tll_R_ref * delta_ref

        #    #print 'delta_tll\n', delta_tll
        #    tip_current_T_tll = tfu.tf_as_matrix(self.tf.lookupTransform('torso_lift_link', self.tool_frame, rospy.Time(0)))
        #    #print 'tip_current_T_tll\n', tip_current_T_tll

        #    #Find translation
        #    delta_T = delta_tll.copy()
        #    delta_T[0:3,0:3] = np.eye(3)
        #    tip_T = delta_T * tip_current_T_tll

        #    #Find rotation
        #    tip_R = delta_tll[0:3, 0:3] * tip_current_T_tll[0:3, 0:3]

        #    tip_new = np.matrix(np.eye(4))
        #    tip_new[0:3, 0:3] = tip_R
        #    tip_new[0:3, 3] = tip_T[0:3,3]

        #    #print 'tip_new\n', tip_new
        #    goal_ps = stamp_pose(mat_to_pose(tip_new), 'torso_lift_link')

<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
import requests
import os

from flask_restful import Resource

import os, sys
sys.path.append(os.path.abspath('..'))

from api_gateway.settings import MOVIES_BASE_URL

class BaseForAPIGW():
    def __init__(self):
        self.base_url = MOVIES_BASE_URL

    def response_text_from_request(self, resource):
        response = requests.get(self.base_url + resource)
        text = response.text
        return text

class MovieList(BaseForAPIGW, Resource):
    def get(self):
        resource = '/movies/'
        return 'wow'

def add_resources(api):
    api.add_resource(MovieList, '/movies/')
    return api
<commit_msg>movies list api from backend<commit_after># -*- encoding: utf-8 -*-
import requests
import os

from flask_restful import Resource

import os, sys
sys.path.append(os.path.abspath('..'))

from api_gateway.settings import MOVIES_BASE_URL

class BaseForAPIGW():
    def __init__(self):
        self.base_url = MOVIES_BASE_URL

    def response_text_from_request(self, resource):
        response = requests.get(self.base_url + resource)
        text = response.text
        return text

class MovieList(BaseForAPIGW, Resource):
    def get(self):
        resource = '/movies/'
        text = self.response_text_from_request(resource)
        return text

def add_resources(api):
    api.add_resource(MovieList, '/movies/')
    return api
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

setup(
    name='ubersmith',
    version='0.3.11',
    author='Jason Keene',
    author_email='jasonkeene@gmail.com',
    description='Client library for the Ubersmith API 2.0',
    long_description=open('README.rst').read(),
    packages=find_packages(exclude=['tests']),
    install_requires=open('requirements.txt').read().strip().split('\n'),
    url='https://github.com/jasonkeene/python-ubersmith',
    license='MIT License',
    keywords=['ubersmith'],
    classifiers=[
        'Programming Language :: Python',
        'Development Status :: 1 - Planning',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
    ],
)
<commit_msg>chore(setup): change package name and version<commit_after>from setuptools import setup, find_packages

setup(
    name='ubersmith-python',
    version='0.0.1',
    author='Jason Keene',
    author_email='jasonkeene@gmail.com',
    description='Client library for the Ubersmith API 2.0',
    long_description=open('README.rst').read(),
    packages=find_packages(exclude=['tests']),
    install_requires=open('requirements.txt').read().strip().split('\n'),
    url='https://github.com/jasonkeene/python-ubersmith',
    license='MIT License',
    keywords=['ubersmith'],
    classifiers=[
        'Programming Language :: Python',
        'Development Status :: 1 - Planning',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
    ],
)
<|endoftext|>


<commit_before># flake8: noqa
from .predicate import P

__version__ = '1.2.2'
__all__ = ['P']

<commit_msg>Bump version to reflect new debugging/validation features<commit_after># flake8: noqa
from .predicate import P

__version__ = '1.3.0'
__all__ = ['P']

<|endoftext|>


<commit_before>"""
.. module:: bottle_utils.form
   :synopsis: Form processing and validation library
.. moduleauthor:: Outernet Inc <hello@outernet.is>
"""
from .exceptions import ValidationError
from .fields import (DormantField,
                     Field,
                     StringField,
                     PasswordField,
                     HiddenField,
                     EmailField,
                     TextAreaField,
                     DateField,
                     FileField,
                     IntegerField,
                     FloatField,
                     BooleanField,
                     SelectField)
from .forms import Form
from .validators import Validator, Required, DateValidator, InRangeValidator

__all__ = ['ValidationError',
           'DormantField',
           'Field',
           'StringField',
           'PasswordField',
           'HiddenField',
           'EmailField',
           'TextAreaField',
           'DateField',
           'FileField',
           'IntegerField',
           'FloatField',
           'BooleanField',
           'SelectField',
           'Form',
           'Validator',
           'Required',
           'DateValidator',
           'InRangeValidator']
<commit_msg>Include LengthValidator in list of exporeted objects<commit_after>"""
.. module:: bottle_utils.form
   :synopsis: Form processing and validation library
.. moduleauthor:: Outernet Inc <hello@outernet.is>
"""
from .exceptions import ValidationError
from .fields import (DormantField,
                     Field,
                     StringField,
                     PasswordField,
                     HiddenField,
                     EmailField,
                     TextAreaField,
                     DateField,
                     FileField,
                     IntegerField,
                     FloatField,
                     BooleanField,
                     SelectField)
from .forms import Form
from .validators import (Validator, Required, DateValidator, InRangeValidator,
                         LengthValidator)

__all__ = ['ValidationError',
           'DormantField',
           'Field',
           'StringField',
           'PasswordField',
           'HiddenField',
           'EmailField',
           'TextAreaField',
           'DateField',
           'FileField',
           'IntegerField',
           'FloatField',
           'BooleanField',
           'SelectField',
           'Form',
           'Validator',
           'Required',
           'DateValidator',
           'InRangeValidator']
<|endoftext|>


<commit_before>#! /usr/local/bin/python
"""
See LICENSE file for copyright and license details.
"""

from modules.constant import *
from generic.modules.calculator_finance import *
from decimal import Decimal
from generic.modules.function import print_in_columns

class Calculator():
    """
        Calculator class
    """

    def __init__(self, pool, amount, tax, commission, shares, price, buy, market, commodity, account, risk):
        """
            Initialize
        """
        self.pool = pool
        self.amount = amount
        self.tax = tax
        self.commission = commission
        self.shares = shares
        self.price = price
        self.buy = buy
        self.market = market
        self.commodity = commodity
        self.account = account
        self.risk = risk
        # result
        self.result_general = {}
        self.result_buy = {}
        self.result_sell = {}

    def calculate(self):
        """
            Calculate all possible unknown values.
        """
        try:
            #Note: the order is important...
            # Input values
            if self.shares == Decimal(-1.0):
                # here, pool, risk, commission and tax and price should be given
                needed = []
                needed.append(self.pool)
                needed.append(self.risk)
                needed.append(self.commission)
                needed.append(self.tax)
                needed.append(self.price)
                if has_missing_parameter(needed):
                    raise Exception("Shares unknown, but parameter(s) missing for calculation.\n" \
                    "Please enter pool, risk, commission, tax and price info.")
                self.shares = calculate_shares_recommended(self.pool, self.risk, self.commission, self.tax, self.price)
            if self.price == Decimal(-1.0):
                # here, amount, shares, tax and commission should be given
                needed = []
                needed.append(self.amount)
                needed.append(self.shares)
                needed.append(self.tax)
                needed.append(self.commission)
                if has_missing_parameter(needed):
                    raise Exception("Price unknown, but parameter(s) missing for calculation.\n" \
                    "Please enter amount, shares, tax and commission info.")
                if self.buy:
                    self.price = calculate_price(
                        Transaction.BUY
                        , self.amount
                        , self.shares
                        , self.tax
                        , self.commission)
                else:
                    self.price = calculate_price(
                        Transaction.SELL
                        , self.amount
                        , self.shares
                        , self.tax
                        , self.commission)
            if self.commission == Decimal(-1.0):
                # here, account, market, commodity, price and shares should be given
                needed = []
                needed.append(self.account)
                needed.append(self.market)
                needed.append(self.commodity)
                needed.append(self.price)
                needed.append(self.shares)
                if has_missing_parameter(needed):
                    raise Exception("Commission unknown, but parameter(s) missing for calculation.\n" \
                    "Please enter account, market, commodity, price and shares info.")
                self.commission = calculate_commission(
                    self.account
                    , self.market
                    , self.commodity
                    , self.price
                    , self.shares)
            if self.amount == Decimal(-1.0):
                # here, price, shares, tax and commission should be given
                needed = []
                needed.append(self.price)
                needed.append(self.shares)
                needed.append(self.tax)
                needed.append(self.commission)
                if has_missing_parameter(needed):
                    raise Exception("Amount unknown, but parameter(s) missing for calculation.\n" \
                    "Please enter price, shares, tax and commission info.")
                if self.buy:
                    self.amount = calculate_amount(
                        self.price
                        , self.shares
                        , Transaction.BUY
                        , self.tax
                        , self.commission)
                else:
                    self.amount = calculate_amount(
                        self.price
                        , self.shares
                        , Transaction.SELL
                        , self.tax
                        , self.commission)

            # Extra calculatable fields
            # TEST INFO
            print(self.amount)
            print(self.tax)
            print(self.commission)
            print(self.shares)
            # /TEST INFO
            # GENERAL - input
            self.result_general["amount"] = str(self.amount)
            self.result_general["tax"] = str(self.tax)
            self.result_general["commission"] = str(self.commission)
            # GENERAL - extra info
            self.result_general["amount_simple"] = str(calculate_amount_simple(self.shares, self.price))
            self.result_general["shares"] = str(calculate_shares_recommended(self.pool, self.risk, self.commission, self.tax, self.price))
            # BUY
            self.result_buy["cost_tax"] = str(cost_tax(Transaction.BUY, self.amount, self.commission, self.shares, self.price))
            self.result_buy["amount_with_tax"] = str(calculate_amount_with_tax(self.tax, self.shares, self.price))
            # SELL
            self.result_sell["cost_tax"] = str(cost_tax(Transaction.SELL, self.amount, self.commission, self.shares, self.price))
            self.result_sell["amount_with_tax"] = str(calculate_amount_with_tax(self.tax, self.shares, self.price))
        except Exception as ex:
            print('Error in calculate:', ex)

    def print_general(self):
        """
            Print the results with headers etc.
        """
        try:
            headers_general = [
                ["amount", "tax", "commission", "shares", "amount_simple"]
                , ['-'*len("amount"), '-'*len("tax"), '-'*len("commission"), '-'*len("shares"), '-'*len("amount_simple")]]
            headers_buy_sell = [
                ["cost_tax", "amount_with_tax"]
                , ['-'*len("cost_tax"), '-'*len("amount_with_tax")]]

            subheader = [["GENERAL"], ['-'*len("GENERAL")*2]]
            print_in_columns(subheader)
            print_in_columns(headers_general)
            print_in_columns([self.result_general.values()])
            if self.buy:
                subheader = [["BUY"], ['-'*len("BUY")*2]]
                print_in_columns(subheader)
                print_in_columns(headers_buy_sell)
                print_in_columns([self.result_buy.values()])
            else:
                subheader = [["SELL"], ['-'*len("SELL")*2]]
                print_in_columns(subheader)
                print_in_columns(headers_buy_sell)
                print_in_columns([self.result_sell.values()])
        except Exception as ex:
            print('Error in print_results():', ex)

    def print_gnucash(self):
        """
            Print statements to enter in gnucash.
        """
        try:
            headers = [
                ['account', 'shares', 'price', 'debit', 'credit']
                , ['-'*len('account'), '-'*len('shares'), '-'*len('price'), '-'*len('debit'), '-'*len('credit')]]
            if self.buy:
                lines = []
                lines.append(["assets:stock:<market>.<commodity>", self.shares, self.price, "", self.result_general["amount_simple"]])
                lines.append(["expenses:commission:stock:<market>.<commodity>", "", "", self.commission])
                lines.append(["expenses:tax:stock:<market>.<commodity>", "", "", self.result_buy["cost_tax"]])
                lines.append(["assets:current_assets:stock:<bank account>", "", "", "", self.amount])
                subheader = [["BUY"], ['-'*len("BUY")*2]]
                print_in_columns(subheader)
                print_in_columns(headers)
                print_in_columns(lines)
            else:
                lines = []
                lines.append(["assets:stock:<market>.<commodity>", self.shares, self.price, self.result_general["amount_simple"], ""])
                lines.append(["expenses:commission:stock:<market>.<commodity>", "", "", self.commission, ""])
                lines.append(["expenses:tax:stock:<market>.<commodity>", "", "", "", self.result_sell["cost_tax"]])
                lines.append(["assets:current_assets:stock:<bank account>", "", "", "", self.amount])
                subheader = [["SELL"], ['-'*len("SELL")*2]]
                print_in_columns(subheader)
                print_in_columns(headers)
                print_in_columns(lines)
        except Exception as ex:
            print('Error in print_gnucash():', ex)

    def has_missing_parameter(self, parameters):
        """
            Returns a boolean that says if the parameters list has a value
            of DEFAULT_DECIMAL or not. True = a parameter is missing.
        """
        return DEFAULT_DECIMAL in parameters
<commit_msg>Moved print functions.<commit_after>#! /usr/local/bin/python
"""
See LICENSE file for copyright and license details.
"""

from modules.constant import *
from generic.modules.calculator_finance import *
from decimal import Decimal
from generic.modules.function import print_in_columns

class Calculator():
    """
        Calculator class
    """

    def __init__(self, pool, amount, tax, commission, shares, price, buy, market, commodity, account, risk):
        """
            Initialize
        """
        self.pool = pool
        self.amount = amount
        self.tax = tax
        self.commission = commission
        self.shares = shares
        self.price = price
        self.buy = buy
        self.market = market
        self.commodity = commodity
        self.account = account
        self.risk = risk
        # result
        self.result_general = {}
        self.result_buy = {}
        self.result_sell = {}

    def calculate(self):
        """
            Calculate all possible unknown values.
        """
        try:
            #Note: the order is important...
            # Input values
            if self.shares == Decimal(-1.0):
                # here, pool, risk, commission and tax and price should be given
                needed = []
                needed.append(self.pool)
                needed.append(self.risk)
                needed.append(self.commission)
                needed.append(self.tax)
                needed.append(self.price)
                if has_missing_parameter(needed):
                    raise Exception("Shares unknown, but parameter(s) missing for calculation.\n" \
                    "Please enter pool, risk, commission, tax and price info.")
                self.shares = calculate_shares_recommended(self.pool, self.risk, self.commission, self.tax, self.price)
            if self.price == Decimal(-1.0):
                # here, amount, shares, tax and commission should be given
                needed = []
                needed.append(self.amount)
                needed.append(self.shares)
                needed.append(self.tax)
                needed.append(self.commission)
                if has_missing_parameter(needed):
                    raise Exception("Price unknown, but parameter(s) missing for calculation.\n" \
                    "Please enter amount, shares, tax and commission info.")
                if self.buy:
                    self.price = calculate_price(
                        Transaction.BUY
                        , self.amount
                        , self.shares
                        , self.tax
                        , self.commission)
                else:
                    self.price = calculate_price(
                        Transaction.SELL
                        , self.amount
                        , self.shares
                        , self.tax
                        , self.commission)
            if self.commission == Decimal(-1.0):
                # here, account, market, commodity, price and shares should be given
                needed = []
                needed.append(self.account)
                needed.append(self.market)
                needed.append(self.commodity)
                needed.append(self.price)
                needed.append(self.shares)
                if has_missing_parameter(needed):
                    raise Exception("Commission unknown, but parameter(s) missing for calculation.\n" \
                    "Please enter account, market, commodity, price and shares info.")
                self.commission = calculate_commission(
                    self.account
                    , self.market
                    , self.commodity
                    , self.price
                    , self.shares)
            if self.amount == Decimal(-1.0):
                # here, price, shares, tax and commission should be given
                needed = []
                needed.append(self.price)
                needed.append(self.shares)
                needed.append(self.tax)
                needed.append(self.commission)
                if has_missing_parameter(needed):
                    raise Exception("Amount unknown, but parameter(s) missing for calculation.\n" \
                    "Please enter price, shares, tax and commission info.")
                if self.buy:
                    self.amount = calculate_amount(
                        self.price
                        , self.shares
                        , Transaction.BUY
                        , self.tax
                        , self.commission)
                else:
                    self.amount = calculate_amount(
                        self.price
                        , self.shares
                        , Transaction.SELL
                        , self.tax
                        , self.commission)

            # Extra calculatable fields
            # TEST INFO
            print(self.amount)
            print(self.tax)
            print(self.commission)
            print(self.shares)
            # /TEST INFO
            # GENERAL - input
            self.result_general["amount"] = str(self.amount)
            self.result_general["tax"] = str(self.tax)
            self.result_general["commission"] = str(self.commission)
            # GENERAL - extra info
            self.result_general["amount_simple"] = str(calculate_amount_simple(self.shares, self.price))
            self.result_general["shares"] = str(calculate_shares_recommended(self.pool, self.risk, self.commission, self.tax, self.price))
            # BUY
            self.result_buy["cost_tax"] = str(cost_tax(Transaction.BUY, self.amount, self.commission, self.shares, self.price))
            self.result_buy["amount_with_tax"] = str(calculate_amount_with_tax(self.tax, self.shares, self.price))
            # SELL
            self.result_sell["cost_tax"] = str(cost_tax(Transaction.SELL, self.amount, self.commission, self.shares, self.price))
            self.result_sell["amount_with_tax"] = str(calculate_amount_with_tax(self.tax, self.shares, self.price))
        except Exception as ex:
            print('Error in calculate:', ex)

    def has_missing_parameter(self, parameters):
        """
            Returns a boolean that says if the parameters list has a value
            of DEFAULT_DECIMAL or not. True = a parameter is missing.
        """
        return DEFAULT_DECIMAL in parameters
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
from cStringIO import StringIO
import logging
from math import ceil
from multiprocessing import Pool
import time
import urlparse

import boto

parser = argparse.ArgumentParser(description="Transfer large files to S3",
        prog="s3-mp-upload")
parser.add_argument("src", type=file, help="The file to transfer")
parser.add_argument("dest", help="The S3 destination object")
parser.add_argument("-np", "--num-processes", help="Number of processors to use",
        type=int, default=2)
parser.add_argument("-f", "--force", help="Overwrite an existing S3 key",
        action="store_true")
parser.add_argument("-s", "--split", help="Split size, in Mb", type=int, default=50)

logger = logging.getLogger("s3-mp-upload")

def do_part_upload(args):
    """
    Upload a part of a MultiPartUpload

    Open the target file and read in a chunk. Since we can't pickle
    S3Connection or MultiPartUpload objects, we have to reconnect and lookup
    the MPU object with each part upload.

    :type args: tuple of (string, string, string, int, int)
    :param args: The actual arguments of this method. Due to lameness of
                 multiprocessing, we have to extract these outside of the
                 function definition.

                 The arguments are: S3 Bucket name, MultiPartUpload id, file
                 name, chunk size, and part number
    """
    # Multiprocessing args lameness
    bucket_name, mpu_id, fname, size, i = args

    # Connect to S3, get the MultiPartUpload
    s3 = boto.connect_s3()
    bucket = s3.lookup(bucket_name)
    mpu = None
    for mp in bucket.list_multipart_uploads():
        if mp.id == mpu_id:
            mpu = mp
            break
    if mpu is None:
        raise Exception("Could not find MultiPartUpload %s" % mpu_id)

    # Read the chunk from the file
    fp = open(fname, 'rb')
    fp.seek(size*i)
    data = fp.read(size)
    fp.close()
    if not data:
        raise Exception("Unexpectedly tried to read an empty chunk")

    # Do the upload
    t1 = time.time()
    mpu.upload_part_from_file(StringIO(data), i+1)

    # Print some timings
    t2 = time.time() - t1
    s = len(data)/1024./1024.
    logger.info("Uploaded part %s (%0.2fM) in %0.2fs at %0.2fMbps" % (i+1, s, t2,
        s/t2))

def main():
    logging.basicConfig(level=logging.INFO)
    args = parser.parse_args()
    logger.debug("Got args: %s" % args)

    # Check that dest is a valid S3 url
    split_rs = urlparse.urlsplit(args.dest)
    if split_rs.scheme != "s3":
        raise ValueError("'%s' is not an S3 url" % args.dest)

    s3 = boto.connect_s3()
    bucket = s3.lookup(split_rs.netloc)
    key = bucket.get_key(split_rs.path)
    # See if we're overwriting an existing key
    if key is not None:
        if not args.force:
            raise ValueError("'%s' already exists. Specify -f to overwrite it" %
                args.dest)

    # Determine the splits
    part_size = 1024*1024*args.split
    args.src.seek(0,2)
    size = args.src.tell()
    num_parts = int(ceil(size / part_size))

    # Create the multi-part upload object
    mpu = bucket.initiate_multipart_upload(split_rs.path)
    def it(num_parts):
        for i in range(num_parts+1):
            yield (bucket.name, mpu.id, args.src.name, part_size, i)
    try:
        # Create a pool of workers
        pool = Pool(processes=args.num_processes)
        t1 = time.time()
        pool.map(do_part_upload, it(num_parts))
        # Print out some timings
        t2 = time.time() - t1
        s = size/1024./1024.
        logger.info("Finished uploading %0.2fM in %0.2fs (%0.2fMbps)" % (s, t2, s/t2))
        # Finalize
        args.src.close()
        mpu.complete_upload()
    except Exception, err:
        logger.error(err)
        mpu.cancel_upload()

if __name__ == "__main__":
    main()
<commit_msg>Adding support for Reduced Redundancy storage.<commit_after>#!/usr/bin/env python
import argparse
from cStringIO import StringIO
import logging
from math import ceil
from multiprocessing import Pool
import time
import urlparse

import boto

parser = argparse.ArgumentParser(description="Transfer large files to S3",
        prog="s3-mp-upload")
parser.add_argument("src", type=file, help="The file to transfer")
parser.add_argument("dest", help="The S3 destination object")
parser.add_argument("-np", "--num-processes", help="Number of processors to use",
        type=int, default=2)
parser.add_argument("-f", "--force", help="Overwrite an existing S3 key",
        action="store_true")
parser.add_argument("-s", "--split", help="Split size, in Mb", type=int, default=50)
parser.add_argument("-rrs", "--reduced-redundancy", help="Use reduced redundancy storage. Default is standard.", default=False,  action="store_true")

logger = logging.getLogger("s3-mp-upload")

def do_part_upload(args):
    """
    Upload a part of a MultiPartUpload

    Open the target file and read in a chunk. Since we can't pickle
    S3Connection or MultiPartUpload objects, we have to reconnect and lookup
    the MPU object with each part upload.

    :type args: tuple of (string, string, string, int, int)
    :param args: The actual arguments of this method. Due to lameness of
                 multiprocessing, we have to extract these outside of the
                 function definition.

                 The arguments are: S3 Bucket name, MultiPartUpload id, file
                 name, chunk size, and part number
    """
    # Multiprocessing args lameness
    bucket_name, mpu_id, fname, size, i = args

    # Connect to S3, get the MultiPartUpload
    s3 = boto.connect_s3()
    bucket = s3.lookup(bucket_name)
    mpu = None
    for mp in bucket.list_multipart_uploads():
        if mp.id == mpu_id:
            mpu = mp
            break
    if mpu is None:
        raise Exception("Could not find MultiPartUpload %s" % mpu_id)

    # Read the chunk from the file
    fp = open(fname, 'rb')
    fp.seek(size*i)
    data = fp.read(size)
    fp.close()
    if not data:
        raise Exception("Unexpectedly tried to read an empty chunk")

    # Do the upload
    t1 = time.time()
    mpu.upload_part_from_file(StringIO(data), i+1)

    # Print some timings
    t2 = time.time() - t1
    s = len(data)/1024./1024.
    logger.info("Uploaded part %s (%0.2fM) in %0.2fs at %0.2fMbps" % (i+1, s, t2,
        s/t2))

def main():
    logging.basicConfig(level=logging.INFO)
    args = parser.parse_args()
    logger.debug("Got args: %s" % args)

    # Check that dest is a valid S3 url
    split_rs = urlparse.urlsplit(args.dest)
    if split_rs.scheme != "s3":
        raise ValueError("'%s' is not an S3 url" % args.dest)

    s3 = boto.connect_s3()
    bucket = s3.lookup(split_rs.netloc)
    key = bucket.get_key(split_rs.path)
    # See if we're overwriting an existing key
    if key is not None:
        if not args.force:
            raise ValueError("'%s' already exists. Specify -f to overwrite it" %
                args.dest)

    # Determine the splits
    part_size = 1024*1024*args.split
    args.src.seek(0,2)
    size = args.src.tell()
    num_parts = int(ceil(size / part_size))

    # Create the multi-part upload object
    mpu = bucket.initiate_multipart_upload(split_rs.path, reduced_redundancy=args.reduced_redundancy)
    def it(num_parts):
        for i in range(num_parts+1):
            yield (bucket.name, mpu.id, args.src.name, part_size, i)
    try:
        # Create a pool of workers
        pool = Pool(processes=args.num_processes)
        t1 = time.time()
        pool.map(do_part_upload, it(num_parts))
        # Print out some timings
        t2 = time.time() - t1
        s = size/1024./1024.
        logger.info("Finished uploading %0.2fM in %0.2fs (%0.2fMbps)" % (s, t2, s/t2))
        # Finalize
        args.src.close()
        mpu.complete_upload()
    except Exception, err:
        logger.error(err)
        mpu.cancel_upload()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
A Pylearn2 Dataset class for accessing the data for the
facial expression recognition Kaggle contest for the ICML
2013 workshop on representation learning.
"""
__authors__ = "Ian Goodfellow"
__copyright__ = "Copyright 2013, Universite de Montreal"
__credits__ = ["Ian Goodfellow"]
__license__ = "3-clause BSD"
__maintainer__ = "Ian Goodfellow"

import csv
import numpy as np
import os

from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.utils import serial
from pylearn2.utils.string_utils import preprocess

class BlackBoxDataset(DenseDesignMatrix):
    """
    A Pylearn2 Dataset class for accessing the data for the
    facial expression recognition Kaggle contest for the ICML
    2013 workshop on representation learning.
    """

    def __init__(self, which_set,
            base_path = '${PYLEARN2_DATA_PATH}/icml_2013_black_box',
            start = None,
            stop = None,
            preprocessor = None,
            fit_preprocessor = False,
            fit_test_preprocessor = False):
        """
        which_set: A string specifying which portion of the dataset
            to load. Valid values are 'train' or 'public_test'
        base_path: The directory containing the .csv files from kaggle.com.
                This directory should be writable; if the .csv files haven't
                already been converted to npy, this class will convert them
                to save memory the next time they are loaded.
        fit_preprocessor: True if the preprocessor is allowed to fit the
                   data.
        fit_test_preprocessor: If we construct a test set based on this
                    dataset, should it be allowed to fit the test set?
        """

        self.test_args = locals()
        self.test_args['which_set'] = 'public_test'
        self.test_args['fit_preprocessor'] = fit_test_preprocessor
        del self.test_args['start']
        del self.test_args['stop']
        del self.test_args['self']

        files = {'train': 'train.csv', 'public_test' : 'test.csv'}

        if which_set == 'extra':
            path = base_path + '/' + 'extra_unsupervised_data.npy'
            X = serial.load(path).T
            y = None
        else:
            try:
                filename = files[which_set]
            except KeyError:
                raise ValueError("Unrecognized dataset name: " + which_set)

            path = base_path + '/' + filename

            path = preprocess(path)

            X, y = self._load_data(path, which_set == 'train')

        if start is not None:
            assert which_set != 'test'
            assert isinstance(start, int)
            assert isinstance(stop, int)
            assert start >= 0
            assert start < stop
            assert stop <= X.shape[0]
            X = X[start:stop, :]
            if y is not None:
                y = y[start:stop, :]

        super(BlackBoxDataset, self).__init__(X=X, y=y)

        if preprocessor:
            preprocessor.apply(self, can_fit=fit_preprocessor)

    def adjust_for_viewer(self, X):
        return (X - 127.5) / 127.5

    def get_test_set(self):
        return BlackBoxDataset(**self.test_args)

    def _load_data(self, path, expect_labels):

        assert path.endswith('.csv')

        # If a previous call to this method has already converted
        # the data to numpy format, load the numpy directly
        X_path = path[:-4] + '.X.npy'
        Y_path = path[:-4] + '.Y.npy'
        if os.path.exists(X_path):
            X = np.load(X_path)
            if expect_labels:
                y = np.load(Y_path)
            else:
                y = None
            return X, y

        # Convert the .csv file to numpy
        csv_file = open(path, 'r')

        reader = csv.reader(csv_file)

        # Discard header
        row = reader.next()

        y_list = []
        X_list = []

        for row in reader:
            if expect_labels:
                y_str = row[0]
                row = row[1:]
                y = int(float(y_str))
                y_list.append(y)
            X_row = map(lambda x: float(x), row)
            X_list.append(X_row)

        X = np.asarray(X_list).astype('float32')
        if expect_labels:
            y = np.asarray(y_list)

            one_hot = np.zeros((y.shape[0],9),dtype='float32')
            for i in xrange(y.shape[0]):
                one_hot[i,y[i] - 1] = 1.
            y = one_hot
        else:
            y = None

        np.save(X_path, X)
        if y is not None:
            np.save(Y_path, y)

        return X, y
<commit_msg>added some input validation to black_box_dataset<commit_after>"""
A Pylearn2 Dataset class for accessing the data for the
facial expression recognition Kaggle contest for the ICML
2013 workshop on representation learning.
"""
__authors__ = "Ian Goodfellow"
__copyright__ = "Copyright 2013, Universite de Montreal"
__credits__ = ["Ian Goodfellow"]
__license__ = "3-clause BSD"
__maintainer__ = "Ian Goodfellow"

import csv
import numpy as np
import os

from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.utils import serial
from pylearn2.utils.string_utils import preprocess

class BlackBoxDataset(DenseDesignMatrix):
    """
    A Pylearn2 Dataset class for accessing the data for the
    facial expression recognition Kaggle contest for the ICML
    2013 workshop on representation learning.
    """

    def __init__(self, which_set,
            base_path = '${PYLEARN2_DATA_PATH}/icml_2013_black_box',
            start = None,
            stop = None,
            preprocessor = None,
            fit_preprocessor = False,
            fit_test_preprocessor = False):
        """
        which_set: A string specifying which portion of the dataset
            to load. Valid values are 'train' or 'public_test'
        base_path: The directory containing the .csv files from kaggle.com.
                This directory should be writable; if the .csv files haven't
                already been converted to npy, this class will convert them
                to save memory the next time they are loaded.
        fit_preprocessor: True if the preprocessor is allowed to fit the
                   data.
        fit_test_preprocessor: If we construct a test set based on this
                    dataset, should it be allowed to fit the test set?
        """

        self.test_args = locals()
        self.test_args['which_set'] = 'public_test'
        self.test_args['fit_preprocessor'] = fit_test_preprocessor
        del self.test_args['start']
        del self.test_args['stop']
        del self.test_args['self']

        files = {'train': 'train.csv', 'public_test' : 'test.csv'}
        sizes = {'train': 1000, 'public_test' : 10000, 'extra': 135735 }

        if which_set == 'extra':
            path = base_path + '/' + 'extra_unsupervised_data.npy'
            X = serial.load(path).T
            y = None
        else:
            try:
                filename = files[which_set]
            except KeyError:
                raise ValueError("Unrecognized dataset name: " + which_set)

            path = base_path + '/' + filename

            path = preprocess(path)

            expect_labels = which_set == 'train'

            X, y = self._load_data(path, expect_labels)
        size = sizes[which_set]
        if X.shape[0] != size:
            raise ValueError("Expected "+str(size)+" examples, got "+str(X.shape[0]))

        if start is not None:
            assert which_set != 'test'
            assert isinstance(start, int)
            assert isinstance(stop, int)
            assert start >= 0
            assert start < stop
            if not (stop <= X.shape[0]):
                raise ValueError("stop must be less than the # of examples but " +
                        "stop is " + str(stop) + " and there are " + str(X.shape[0]) +
                        " examples.")
            X = X[start:stop, :]
            if y is not None:
                y = y[start:stop, :]

        super(BlackBoxDataset, self).__init__(X=X, y=y)

        if preprocessor:
            preprocessor.apply(self, can_fit=fit_preprocessor)

    def adjust_for_viewer(self, X):
        return (X - 127.5) / 127.5

    def get_test_set(self):
        return BlackBoxDataset(**self.test_args)

    def _load_data(self, path, expect_labels):

        assert path.endswith('.csv')

        # If a previous call to this method has already converted
        # the data to numpy format, load the numpy directly
        X_path = path[:-4] + '.X.npy'
        Y_path = path[:-4] + '.Y.npy'
        if os.path.exists(X_path):
            X = np.load(X_path)
            if expect_labels:
                y = np.load(Y_path)
            else:
                y = None
            return X, y

        # Convert the .csv file to numpy
        csv_file = open(path, 'r')

        reader = csv.reader(csv_file)

        # Discard header
        row = reader.next()

        y_list = []
        X_list = []

        for row in reader:
            if expect_labels:
                y_str = row[0]
                row = row[1:]
                y = int(float(y_str))
                y_list.append(y)
            X_row = map(lambda x: float(x), row)
            X_list.append(X_row)

        X = np.asarray(X_list).astype('float32')
        if expect_labels:
            y = np.asarray(y_list)

            one_hot = np.zeros((y.shape[0],9),dtype='float32')
            for i in xrange(y.shape[0]):
                one_hot[i,y[i] - 1] = 1.
            y = one_hot
        else:
            y = None

        np.save(X_path, X)
        if y is not None:
            np.save(Y_path, y)

        return X, y
<|endoftext|>


<commit_before>#! python3
# -*- coding: utf-8 -*-
"""Checks messages in database and sends PM if expiration time passed.

This module checks messages saved in a MySQL-compatible database and sends a reminder
via PM if the expiration time has passed. If so, the message is removed from the
database.
"""

import praw
import praw.exceptions

import logging.config
import datetime
import time

from clashcallerbotreddit.database import ClashCallerDatabase
from clashcallerbotreddit import LOGGING, config

# Logger
logging.config.dictConfig(LOGGING)
logging.raiseExceptions = True  # Production mode if False (no console sys.stderr output)
logger = logging.getLogger('reply')

# Generate reddit instance
reddit = praw.Reddit('clashcallerreply')  # Section name in praw.ini
subreddit = reddit.subreddit('ClashCallerBot')  # Limit scope for testing purposes

# Make database instance
db = ClashCallerDatabase(config, root_user=False)

def main():
    logger.info('Start reply.py...')
    while True:
        time.sleep(120)  # 2 minutes
        db.open_connections()
        # Get list of messages older than current datetime
        now = datetime.datetime.now(datetime.timezone.utc)
        messages = db.get_messages(now)

        if not messages:
            logger.debug(f'No messages before: {now}.')
            db.close_connections()
            continue

        # Send reminder PM
        for message in messages:
            tid, link, msg, _exp, usr = message
            logger.debug(f'Found message: {tid}, {msg}')
            send_reminder(link, msg, usr)
            logger.info(f'Reminder sent: {link}.')

            # Delete message from database
            db.delete_message(tid)
            logger.info(f'Message deleted.')
        db.close_connections()

def send_reminder(link: str, msg: str, usr: str)-> None:
    """Sends reminder PM to username.

    Function sends given permalink and message to given username.

    Args:
         link:  Permalink to comment.
         msg:   Message in comment that was saved.
         usr:   User to send reminder to.

    """
    subject = 'ClashCallerBot Private Message Here!'
    permalink = 'https://np.reddit.com' + link  # Permalinks are missing prefix
    parent = get_parent(link)
    message = f"""**The message:** {msg}
              **The original comment:** {permalink}
              **The parent comment or submission:** {parent}

              Thank you for entrusting us with your warring needs,
              - ClashCallerBot

              [^(More info)](https://www.reddit.com/r/ClashCallerBot/comments/4e9vo7/clashcallerbot_info/)
              """
    try:
        reddit.redditor(usr).message(subject, message.replace('              ', ''))

    except praw.exceptions.PRAWException as err:
        logger.exception(f'send_reminder: {err}')

def get_parent(link: str) -> str:
    """Fetch parent comment or submission.

    Function gets parent comment of given permalink or submission if top level comment.

    Args:
         link:  Permalink to get parent of.

    Returns:
        Parent comment link or default string.
    """
    permalink = 'https://www.reddit.com' + link  # Permalink is missing prefix
    parent = 'Parent comment not found.'  # Default string
    try:
        comment = reddit.comment(url=permalink)  # Fetch comment by URL
        parent = 'https://np.reddit.com' + comment.submission.permalink
    except praw.exceptions as err:
        logger.exception(f'get_parent: {err}')
    return parent

# If run directly, instead of imported as a module, run main():
if __name__ == '__main__':
    main()
<commit_msg>Update reply.py refactored main function into new function check_database<commit_after>#! python3
# -*- coding: utf-8 -*-
"""Checks messages in database and sends PM if expiration time passed.

This module checks messages saved in a MySQL-compatible database and sends a reminder
via PM if the expiration time has passed. If so, the message is removed from the
database.
"""

import praw
import praw.exceptions

import logging.config
import datetime
import time

from clashcallerbotreddit.database import ClashCallerDatabase
from clashcallerbotreddit import LOGGING, config

# Logger
logging.config.dictConfig(LOGGING)
logging.raiseExceptions = True  # Production mode if False (no console sys.stderr output)
logger = logging.getLogger('reply')

# Generate reddit instance
reddit = praw.Reddit('clashcallerreply')  # Section name in praw.ini
subreddit = reddit.subreddit('ClashCallerBot')  # Limit scope for testing purposes

# Make database instance
db = ClashCallerDatabase(config, root_user=False)

def main():
    logger.info('Start reply.py...')
    while True:
        time.sleep(120)  # 2 minutes

        # Check saved messages
        check_database()

def check_database()-> None:
    """Checks messages in database and sends PM if expiration time passed.

    Checks messages saved in a MySQL-compatible database and sends a reminder
    via PM if the expiration time has passed. If so, the message is removed from the
    database.

    Returns:
         None. Message is removed from database if expiration time has passed.
    """
    db.open_connections()
    # Get list of messages older than current datetime
    now = datetime.datetime.now(datetime.timezone.utc)
    messages = db.get_messages(now)

    if not messages:
        logger.debug(f'No messages before: {now}.')
        db.close_connections()
        return None

    # Send reminder PM
    for message in messages:
        tid, link, msg, _exp, usr = message
        logger.debug(f'Found message: {tid}, {msg}')
        send_reminder(link, msg, usr)
        logger.info(f'Reminder sent: {link}.')

        # Delete message from database
        db.delete_message(tid)
        logger.info(f'Message deleted.')
    db.close_connections()

def send_reminder(link: str, msg: str, usr: str)-> None:
    """Sends reminder PM to username.

    Function sends given permalink and message to given username.

    Args:
         link:  Permalink to comment.
         msg:   Message in comment that was saved.
         usr:   User to send reminder to.

    """
    subject = 'ClashCallerBot Private Message Here!'
    permalink = 'https://np.reddit.com' + link  # Permalinks are missing prefix
    parent = get_parent(link)
    message = f"""**The message:** {msg}
              **The original comment:** {permalink}
              **The parent comment or submission:** {parent}

              Thank you for entrusting us with your warring needs,
              - ClashCallerBot

              [^(More info)](https://www.reddit.com/r/ClashCallerBot/comments/4e9vo7/clashcallerbot_info/)
              """
    try:
        reddit.redditor(usr).message(subject, message.replace('              ', ''))

    except praw.exceptions.PRAWException as err:
        logger.exception(f'send_reminder: {err}')

def get_parent(link: str) -> str:
    """Fetch parent comment or submission.

    Function gets parent comment of given permalink or submission if top level comment.

    Args:
         link:  Permalink to get parent of.

    Returns:
        Parent comment link or default string.
    """
    permalink = 'https://www.reddit.com' + link  # Permalink is missing prefix
    parent = 'Parent comment not found.'  # Default string
    try:
        comment = reddit.comment(url=permalink)  # Fetch comment by URL
        parent = 'https://np.reddit.com' + comment.submission.permalink
    except praw.exceptions as err:
        logger.exception(f'get_parent: {err}')
    return parent

# If run directly, instead of imported as a module, run main():
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from contextlib import contextmanager

from mock import patch
from nose.tools import istest

from provy.more.debian import AptitudeRole, MongoDBRole
from tests.unit.tools.helpers import ProvyTestCase

class MongoDBRoleTest(ProvyTestCase):
    def setUp(self):
        super(MongoDBRoleTest, self).setUp()
        self.role = MongoDBRole(prov=None, context={})

    @contextmanager
    def mongo_method(self, method_name):
        with patch('provy.more.debian.MongoDBRole.%s' % method_name) as mock:
            yield mock

    @contextmanager
    def mock_file_ops(self):
        with self.mock_role_method('read_remote_file') as read_remote_file:
            with self.mock_role_method('write_to_temp_file') as write_to_temp_file:
                with self.mock_role_method('put_file') as put_file:
                    yield read_remote_file, write_to_temp_file, put_file

    def content_from_list(self, list_):
        return '\n'.join(list_)

    @istest
    def installs_necessary_packages_to_provision_to_debian(self):
        with self.using_stub(AptitudeRole) as mock_aptitude:
            self.role.provision_to_debian()

            mock_aptitude.ensure_gpg_key.assert_called_with('http://docs.mongodb.org/10gen-gpg-key.asc')
            mock_aptitude.ensure_aptitude_source.assert_called_with('deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen')
            mock_aptitude.force_update.assert_called_with()
            mock_aptitude.ensure_package_installed.assert_called_with('mongodb-10gen')

    @istest
    def installs_necessary_packages_to_provision_to_ubuntu(self):
        with self.using_stub(AptitudeRole) as mock_aptitude:
            self.role.provision_to_ubuntu()

            mock_aptitude.ensure_gpg_key.assert_called_with('http://docs.mongodb.org/10gen-gpg-key.asc')
            mock_aptitude.ensure_aptitude_source.assert_called_with('deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen')
            mock_aptitude.force_update.assert_called_with()
            mock_aptitude.ensure_package_installed.assert_called_with('mongodb-10gen')

    @istest
    def provisions_to_debian_if_is_debian(self):
        with self.provisioning_to('debian'), self.mongo_method('provision_to_debian') as provision_to_debian:
            self.role.provision()
            provision_to_debian.assert_called_with()

    @istest
    def provisions_to_ubuntu_if_is_ubuntu(self):
        with self.provisioning_to('ubuntu'), self.mongo_method('provision_to_ubuntu') as provision_to_ubuntu:
            self.role.provision()
            provision_to_ubuntu.assert_called_with()

    @istest
    def restarts_the_server(self):
        with self.execute_mock() as execute:
            self.role.restart()

            execute.assert_called_with('service mongodb restart', sudo=True)

    @istest
    def appends_configuration_to_server_config(self):
        with self.mock_file_ops() as (read_remote_file, write_to_temp_file, put_file):
            read_remote_file.return_value = self.content_from_list([
                'foo=Foo',
            ])
            write_to_temp_file.return_value = '/some/tmp/path'

            self.role.configure({
                'bar': 'Bar',
            })

            read_remote_file.assert_called_with('/etc/mongodb.conf', sudo=True)
            write_to_temp_file.assert_called_with(self.content_from_list([
                'foo = Foo',
                'bar = Bar',
                '',  # newline in the end of the file
            ]))
            put_file.assert_called_with(from_file='/some/tmp/path', to_file='/etc/mongodb.conf', sudo=True)

    @istest
    def converts_boolean_config_from_input(self):
        with self.mock_file_ops() as (read_remote_file, write_to_temp_file, put_file):
            read_remote_file.return_value = self.content_from_list([
                'foo=Foo',
            ])
            write_to_temp_file.return_value = '/some/tmp/path'

            self.role.configure({
                'bar': True,
            })

            read_remote_file.assert_called_with('/etc/mongodb.conf', sudo=True)
            write_to_temp_file.assert_called_with(self.content_from_list([
                'foo = Foo',
                'bar = true',
                '',  # newline in the end of the file
            ]))
            put_file.assert_called_with(from_file='/some/tmp/path', to_file='/etc/mongodb.conf', sudo=True)

    @istest
    def overwrites_original_configuration_when_redefined(self):
        with self.mock_file_ops() as (read_remote_file, write_to_temp_file, put_file):
            read_remote_file.return_value = self.content_from_list([
                'foo=Foo',
            ])
            write_to_temp_file.return_value = '/some/tmp/path'

            self.role.configure({
                'foo': 'Baz',
                'bar': 'Bar',
            })

            read_remote_file.assert_called_with('/etc/mongodb.conf', sudo=True)
            write_to_temp_file.assert_called_with(self.content_from_list([
                'foo = Baz',
                'bar = Bar',
                '',  # newline in the end of the file
            ]))
            put_file.assert_called_with(from_file='/some/tmp/path', to_file='/etc/mongodb.conf', sudo=True)
<commit_msg>Favoring helper methods for MongoDBRole tests<commit_after>from nose.tools import istest

from provy.more.debian import AptitudeRole, MongoDBRole
from tests.unit.tools.helpers import ProvyTestCase

class MongoDBRoleTest(ProvyTestCase):
    def setUp(self):
        super(MongoDBRoleTest, self).setUp()
        self.role = MongoDBRole(prov=None, context={})

    def content_from_list(self, list_):
        return '\n'.join(list_)

    @istest
    def installs_necessary_packages_to_provision_to_debian(self):
        with self.using_stub(AptitudeRole) as mock_aptitude:
            self.role.provision_to_debian()

            mock_aptitude.ensure_gpg_key.assert_called_with('http://docs.mongodb.org/10gen-gpg-key.asc')
            mock_aptitude.ensure_aptitude_source.assert_called_with('deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen')
            mock_aptitude.force_update.assert_called_with()
            mock_aptitude.ensure_package_installed.assert_called_with('mongodb-10gen')

    @istest
    def installs_necessary_packages_to_provision_to_ubuntu(self):
        with self.using_stub(AptitudeRole) as mock_aptitude:
            self.role.provision_to_ubuntu()

            mock_aptitude.ensure_gpg_key.assert_called_with('http://docs.mongodb.org/10gen-gpg-key.asc')
            mock_aptitude.ensure_aptitude_source.assert_called_with('deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen')
            mock_aptitude.force_update.assert_called_with()
            mock_aptitude.ensure_package_installed.assert_called_with('mongodb-10gen')

    @istest
    def provisions_to_debian_if_is_debian(self):
        with self.provisioning_to('debian'), self.mock_role_method('provision_to_debian') as provision_to_debian:
            self.role.provision()
            provision_to_debian.assert_called_with()

    @istest
    def provisions_to_ubuntu_if_is_ubuntu(self):
        with self.provisioning_to('ubuntu'), self.mock_role_method('provision_to_ubuntu') as provision_to_ubuntu:
            self.role.provision()
            provision_to_ubuntu.assert_called_with()

    @istest
    def restarts_the_server(self):
        with self.execute_mock() as execute:
            self.role.restart()

            execute.assert_called_with('service mongodb restart', sudo=True)

    @istest
    def appends_configuration_to_server_config(self):
        with self.mock_role_methods('read_remote_file', 'write_to_temp_file', 'put_file') as (read_remote_file, write_to_temp_file, put_file):
            read_remote_file.return_value = self.content_from_list([
                'foo=Foo',
            ])
            write_to_temp_file.return_value = '/some/tmp/path'

            self.role.configure({
                'bar': 'Bar',
            })

            read_remote_file.assert_called_with('/etc/mongodb.conf', sudo=True)
            write_to_temp_file.assert_called_with(self.content_from_list([
                'foo = Foo',
                'bar = Bar',
                '',  # newline in the end of the file
            ]))
            put_file.assert_called_with(from_file='/some/tmp/path', to_file='/etc/mongodb.conf', sudo=True)

    @istest
    def converts_boolean_config_from_input(self):
        with self.mock_role_methods('read_remote_file', 'write_to_temp_file', 'put_file') as (read_remote_file, write_to_temp_file, put_file):
            read_remote_file.return_value = self.content_from_list([
                'foo=Foo',
            ])
            write_to_temp_file.return_value = '/some/tmp/path'

            self.role.configure({
                'bar': True,
            })

            read_remote_file.assert_called_with('/etc/mongodb.conf', sudo=True)
            write_to_temp_file.assert_called_with(self.content_from_list([
                'foo = Foo',
                'bar = true',
                '',  # newline in the end of the file
            ]))
            put_file.assert_called_with(from_file='/some/tmp/path', to_file='/etc/mongodb.conf', sudo=True)

    @istest
    def overwrites_original_configuration_when_redefined(self):
        with self.mock_role_methods('read_remote_file', 'write_to_temp_file', 'put_file') as (read_remote_file, write_to_temp_file, put_file):
            read_remote_file.return_value = self.content_from_list([
                'foo=Foo',
            ])
            write_to_temp_file.return_value = '/some/tmp/path'

            self.role.configure({
                'foo': 'Baz',
                'bar': 'Bar',
            })

            read_remote_file.assert_called_with('/etc/mongodb.conf', sudo=True)
            write_to_temp_file.assert_called_with(self.content_from_list([
                'foo = Baz',
                'bar = Bar',
                '',  # newline in the end of the file
            ]))
            put_file.assert_called_with(from_file='/some/tmp/path', to_file='/etc/mongodb.conf', sudo=True)
<|endoftext|>


<commit_before><commit_msg>Delete urls.py<commit_after><|endoftext|>


<commit_before>"""
byceps.blueprints.api.decorators
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2021 Jochen Kupperschmidt
:License: Revised BSD (see `LICENSE` file for details)
"""

from functools import wraps
from typing import Optional

from flask import abort, request
from werkzeug.datastructures import WWWAuthenticate

from ...services.authentication.api import service as api_service

def api_token_required(func):
    """Ensure the request is authenticated via API token."""

    @wraps(func)
    def wrapper(*args, **kwargs):
        if not _has_valid_api_token():
            www_authenticate = WWWAuthenticate('Bearer')
            abort(401, www_authenticate=www_authenticate)
        return func(*args, **kwargs)

    return wrapper

def _has_valid_api_token() -> bool:
    request_token = _extract_token_from_request()
    if request_token is None:
        return False

    api_token = api_service.find_api_token_by_token(request_token)
    return api_token is not None and not api_token.suspended

def _extract_token_from_request() -> Optional[str]:
    header_value = request.headers.get('Authorization')
    if header_value is None:
        return None

    return header_value.replace('Bearer ', '', 1)
<commit_msg>Add `invalid_token` error to `WWW-Authenticate` header if API token is suspended<commit_after>"""
byceps.blueprints.api.decorators
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2021 Jochen Kupperschmidt
:License: Revised BSD (see `LICENSE` file for details)
"""

from functools import wraps
from typing import Optional

from flask import abort, request
from werkzeug.datastructures import WWWAuthenticate

from ...services.authentication.api import service as api_service
from ...services.authentication.api.transfer.models import ApiToken

def api_token_required(func):
    """Ensure the request is authenticated via API token."""

    @wraps(func)
    def wrapper(*args, **kwargs):
        api_token = _find_valid_api_token()

        if api_token is None:
            www_authenticate = WWWAuthenticate('Bearer')
            abort(401, www_authenticate=www_authenticate)

        if api_token.suspended:
            www_authenticate = WWWAuthenticate('Bearer')
            www_authenticate['error'] = 'invalid_token'
            abort(401, www_authenticate=www_authenticate)

        return func(*args, **kwargs)

    return wrapper

def _find_valid_api_token() -> Optional[ApiToken]:
    request_token = _extract_token_from_request()
    if request_token is None:
        return None

    return api_service.find_api_token_by_token(request_token)

def _extract_token_from_request() -> Optional[str]:
    header_value = request.headers.get('Authorization')
    if header_value is None:
        return None

    return header_value.replace('Bearer ', '', 1)
<|endoftext|>


<commit_before>SECRET_KEY = '\xc4\x96\x03\xc8,9z8\x92\xfc\xafi\x03\x96;\xf8\x1e\xcc^\x8e\xc76'

DATABASE = dict(
    DB_BACKEND='postgresql',
    DB_USER='webuser',
    DB_PASS='webpass',
    DB_HOST='localhost',
    DB_NAME='finance',
)

HEADERS_ALLOWED = 'origin, x-requested-with, content-type, accept, authtoken'
<commit_msg>db uri set properly<commit_after>SECRET_KEY = '\xc4\x96\x03\xc8,9z8\x92\xfc\xafi\x03\x96;\xf8\x1e\xcc^\x8e\xc76'
SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://webuser:webpass@localhost/finance'
HEADERS_ALLOWED = 'origin, x-requested-with, content-type, accept, authtoken'
<|endoftext|>