<commit_before>#!/usr/bin/python
# vim: expandtab:tabstop=2:shiftwidth=2
import boto3
from botocore.exceptions import ClientError
# Ansible modules need this wildcard import
# pylint: disable=unused-wildcard-import, wildcard-import, redefined-builtin
from ansible.module_utils.basic import *

''' oo_ec2_elb_set_cert module
    .
Test Playbook:
---
- hosts: localhost
  gather_facts: no
  tasks:
  - include_role:
      name: ../../roles/lib_utils
    static: true

  - oo_ec2_elb_set_cert:
      external_elb_name: 'elb-name'
      aws_iam_openshift_cert_arn: 'certificate id'
      region: "us-east-1"
'''

class AwsElb(object):
    ''' AWS ELB class '''
    def __init__(self):
        self.module = None
        self.elb_client = None

    def get_ssl_cert_arn(self, elb_name):
        elb_client = boto3.client('elb')
        try:
            elb = elb_client.describe_load_balancers(LoadBalancerNames=[elb_name])
        except ClientError as e:
                self.module.fail_json(failed=True,
                                      msg=e)

        for listener in elb['LoadBalancerDescriptions'][0]['ListenerDescriptions']:
            if 'SSLCertificateId' in listener['Listener']:
                return listener['Listener']['SSLCertificateId']

    def set_elb_ssl_cert(self, elb_name, cert_id):
        try:
            response = self.elb_client.set_load_balancer_listener_ssl_certificate(LoadBalancerPort=443,
            SSLCertificateId=cert_id,
            LoadBalancerName=elb_name)
        except ClientError as e:
                code = e.response['Error']['Code']
                msg = e.response['Error']['Code']
                self.module.fail_json(failed=True,
                                      msg='AWS API error (%s) occured in ec2_elb_set_cert ansilbe module\n %s\n' % code % msg)

        return response

    def main(self):
        self.module = AnsibleModule(
            argument_spec=dict(
                external_elb_name=dict(required=True, type='str'),
                aws_iam_openshift_cert_arn=dict(required=True, type='str'),
                region=dict(default=None, required=True, type='str'),
                aws_access_key_id=dict(default=None, type='str'),
                aws_secret_access_key=dict(default=None, type='str'),
            ),
        )

        aws_access_key_id = self.module.params.get('aws_access_key_id')
        aws_secret_access_key = self.module.params.get('aws_secret_access_key')
        external_elb_name = self.module.params.get('external_elb_name')
        SSLCertificateId = self.module.params.get('aws_iam_openshift_cert_arn')

        if aws_access_key_id and aws_secret_access_key:
            boto3.setup_default_session(aws_access_key_id=aws_access_key_id,
                                        aws_secret_access_key=aws_secret_access_key,
                                        region_name=self.module.params['region'])
        else:
            boto3.setup_default_session(region_name=self.module.params['region'])

        self.elb_client = boto3.client('elb')

        if not external_elb_name or not SSLCertificateId:
            self.module.fail_json(failed=True,
                                  msg='external_elb_name or aws_iam_openshift_cert_arn not provided')

        # if cert is already applied exit without errors
        current_cert_id = self.get_ssl_cert_arn(external_elb_name)
        if current_cert_id == SSLCertificateId:
            self.module.exit_json(failed=True, changed=False, msg="SSLCertificateId %s already assigned to load balancer %s" % SSLCertificateId % external_elb_name)

        # set the new cert to the elb
        respone = self.set_elb_ssl_cert(external_elb_name,SSLCertificateId)
        if respone['ResponseMetadata']['HTTPStatusCode'] != 200:
            self.module.fail_json(failed=True, msg='failed to set elb')
        else:
            current_cert_id = self.get_ssl_cert_arn(external_elb_name)
            if current_cert_id == SSLCertificateId:
                self.module.exit_json(changed=True, result=current_cert_id)
            else:
                self.module.fail_json(failed=True,
                                      msg='failed for unknown reason' )

if __name__ == '__main__':
    AwsElb().main()
<commit_msg>fixed format strings<commit_after>#!/usr/bin/python
# vim: expandtab:tabstop=2:shiftwidth=2
import boto3
from botocore.exceptions import ClientError
# Ansible modules need this wildcard import
# pylint: disable=unused-wildcard-import, wildcard-import, redefined-builtin
from ansible.module_utils.basic import *

''' oo_ec2_elb_set_cert module
    .
Test Playbook:
---
- hosts: localhost
  gather_facts: no
  tasks:
  - include_role:
      name: ../../roles/lib_utils
    static: true

  - oo_ec2_elb_set_cert:
      external_elb_name: 'elb-name'
      aws_iam_openshift_cert_arn: 'certificate id'
      region: "us-east-1"
'''

class AwsElb(object):
    ''' AWS ELB class '''
    def __init__(self):
        self.module = None
        self.elb_client = None

    def get_ssl_cert_arn(self, elb_name):
        elb_client = boto3.client('elb')
        try:
            elb = elb_client.describe_load_balancers(LoadBalancerNames=[elb_name])
        except ClientError as e:
                self.module.fail_json(failed=True,
                                      msg=e)

        for listener in elb['LoadBalancerDescriptions'][0]['ListenerDescriptions']:
            if 'SSLCertificateId' in listener['Listener']:
                return listener['Listener']['SSLCertificateId']

    def set_elb_ssl_cert(self, elb_name, cert_id):
        try:
            response = self.elb_client.set_load_balancer_listener_ssl_certificate(LoadBalancerPort=443,
            SSLCertificateId=cert_id,
            LoadBalancerName=elb_name)
        except ClientError as e:
                code = e.response['Error']['Code']
                msg = e.response['Error']['Code']
                self.module.fail_json(failed=True,
                                      msg='AWS API error (%s) occured in ec2_elb_set_cert ansilbe module\n %s\n' %(code, msg))

        return response

    def main(self):
        self.module = AnsibleModule(
            argument_spec=dict(
                external_elb_name=dict(required=True, type='str'),
                aws_iam_openshift_cert_arn=dict(required=True, type='str'),
                region=dict(default=None, required=True, type='str'),
                aws_access_key_id=dict(default=None, type='str'),
                aws_secret_access_key=dict(default=None, type='str'),
            ),
        )

        aws_access_key_id = self.module.params.get('aws_access_key_id')
        aws_secret_access_key = self.module.params.get('aws_secret_access_key')
        external_elb_name = self.module.params.get('external_elb_name')
        SSLCertificateId = self.module.params.get('aws_iam_openshift_cert_arn')

        if aws_access_key_id and aws_secret_access_key:
            boto3.setup_default_session(aws_access_key_id=aws_access_key_id,
                                        aws_secret_access_key=aws_secret_access_key,
                                        region_name=self.module.params['region'])
        else:
            boto3.setup_default_session(region_name=self.module.params['region'])

        self.elb_client = boto3.client('elb')

        if not external_elb_name or not SSLCertificateId:
            self.module.fail_json(failed=True,
                                  msg='external_elb_name or aws_iam_openshift_cert_arn not provided')

        # if cert is already applied exit without errors
        current_cert_id = self.get_ssl_cert_arn(external_elb_name)
        if current_cert_id == SSLCertificateId:
            self.module.exit_json(failed=True, changed=False, msg="SSLCertificateId %s already assigned to load balancer %s" % (SSLCertificateId, external_elb_name))

        # set the new cert to the elb
        respone = self.set_elb_ssl_cert(external_elb_name,SSLCertificateId)
        if respone['ResponseMetadata']['HTTPStatusCode'] != 200:
            self.module.fail_json(failed=True, msg='failed to set elb')
        else:
            current_cert_id = self.get_ssl_cert_arn(external_elb_name)
            if current_cert_id == SSLCertificateId:
                self.module.exit_json(changed=True, result=current_cert_id)
            else:
                self.module.fail_json(failed=True,
                                      msg='failed for unknown reason' )

if __name__ == '__main__':
    AwsElb().main()
<|endoftext|>


<commit_before>import os
import json
import timeit
from ..integrated_image_processing import IntegratedImageProcessing
from ..Classifiers import *
from .settings import Settings
from .logger import Logger

class IntegratedImageProcessingTester(object):

    @staticmethod
    def complete_integrated_image_processing():
        if (os.path.isdir(Settings.IMAGE_PROCESSING_REPORT_SAVE_PATH)):
            raise Exception("Cannot create Image Processing Report Save Path: Save directory already exists")
        os.mkdir(Settings.IMAGE_PROCESSING_REPORT_SAVE_PATH)

        if (os.path.isdir(Settings.IMAGE_SAVE_PATH)):
            raise Exception("Cannot create Image Save Path: Save directory already exists")
        os.mkdir(Settings.IMAGE_SAVE_PATH)

        if (os.path.isdir(Settings.JSON_SAVE_PATH)):
            raise Exception("Cannot create Json Save Path: Save directory already exists")
        os.mkdir(Settings.JSON_SAVE_PATH)

        IntegratedImageProcessing.run_integrated_image_processing(Settings.TARGET_MAP_PATH, Settings.IMAGE_SAVE_PATH, Settings.JSON_SAVE_PATH)

    @staticmethod
    def run_integrated_image_processing_tester():
        """
        Compare the image processing result with the answers of the target map
        and print out the results.
        """
        number_of_target_maps = len(os.listdir(Settings.JSON_SAVE_PATH))

        overall_true_positive_count = 0
        overall_false_positive_count = 0

        true_shape_count = 0
        false_shape_count = 0

        true_shape_color_count = 0
        false_shape_color_count = 0

        true_letter_color_count = 0
        false_letter_color_count = 0

        overall_target_count = 0

        for index_0 in range(number_of_target_maps):

            answer_sheet = json.load(open(os.path.join(Settings.TARGET_MAP_ANSWER_SHEET_PATH, str(index_0 + 1) + ".json")))
            answer_list = []

            for index_1 in range(len(answer_sheet["targets"])):
                answer_list.append((answer_sheet["targets"][index_1]))

            overall_target_count += len(answer_list)

            target_detection_result = json.load(open(os.path.join(Settings.JSON_SAVE_PATH, str(index_0 + 1) + ".jpg.json")))
            result_list = []

            for index_2 in range(len(target_detection_result["image_processing_results"])):
                result_list.append((target_detection_result["image_processing_results"][index_2]))

            current_true_positive_count = 0
            current_false_positive_count = 0

            banned_index_list = []

            for index_3 in range(len(answer_list)):
                true_positive_found = False

                for index_4 in range(len(result_list)):
                    is_index_4_banned = False

                    for index_5 in range(len(banned_index_list)):
                        if (index_4 == banned_index_list[index_5]):
                            is_index_4_banned = True

                    if is_index_4_banned == True:
                        continue

                    correct_target_center_x = answer_list[index_3]["target_center_coordinates"][0]
                    correct_target_center_y = answer_list[index_3]["target_center_coordinates"][1]

                    detected_target_center_x = result_list[index_4]["target_location"][0]
                    detected_target_center_y = result_list[index_4]["target_location"][1]

                    if (abs(correct_target_center_x - detected_target_center_x) <= 20) and (abs(correct_target_center_y - detected_target_center_y) <= 20):
                        current_true_positive_count += 1
                        banned_index_list.append(index_4)
                        true_positive_found = True

                        current_target_shape = answer_list[index_3]["shape_type"]
                        detected_target_shape = result_list[index_4]["target_shape_type"]

                        current_target_shape_color = ColorClassifier().convert_rgba_to_color_name(answer_list[index_3]["shape_color"])
                        detected_target_shape_color = result_list[index_4]["target_shape_color"]

                        current_target_letter_color = ColorClassifier().convert_rgba_to_color_name(answer_list[index_3]["alphanumeric_color"])
                        detected_target_letter_color = result_list[index_4]["target_letter_color"]

                        if current_target_shape == detected_target_shape:
                            true_shape_count += 1
                        else:
                            false_shape_count += 1

                        if current_target_shape_color == detected_target_shape_color:
                            true_shape_color_count += 1
                        else:
                            false_shape_color_count += 1

                        if current_target_letter_color == detected_target_letter_color:
                            true_letter_color_count += 1
                        else:
                            false_letter_color_count += 1

                        continue

            current_false_positive_count = len(result_list) - current_true_positive_count

            overall_true_positive_count += current_true_positive_count
            overall_false_positive_count += current_false_positive_count

        target_detection_percentage = 100 * float(overall_true_positive_count) / (overall_target_count)
        shape_detection_percentage = 100 * float(true_shape_count) / (overall_true_positive_count)
        shape_color_detection_percentage = 100 * float(true_shape_color_count) / (overall_true_positive_count)
        letter_color_detection_percentage = 100 * float(true_letter_color_count) / (overall_true_positive_count)

        Logger.log("--------------------------------------------------")
        Logger.log("Total True Positive Count: " + str(overall_true_positive_count))
        Logger.log("Total False Positive Count: " + str(overall_false_positive_count))
        Logger.log("Percentage of Successfully Detected Targets: " + str(target_detection_percentage) + "%")
        Logger.log("--------------------------------------------------")
        Logger.log("Total True Shape Count: " + str(true_shape_count))
        Logger.log("Total False Shape Count: " + str(false_shape_count))
        Logger.log("Percentage of Successfully Detected : " + str(shape_detection_percentage) + "%")
        Logger.log("--------------------------------------------------")
        Logger.log("Total True Shape Color Count: " + str(true_shape_color_count))
        Logger.log("Total False Shape Color Count: " + str(false_shape_color_count))
        Logger.log("Percentage of Successfully Detected : " + str(shape_color_detection_percentage) + "%")
        Logger.log("--------------------------------------------------")
        Logger.log("Total True Letter Color Count: " + str(true_letter_color_count))
        Logger.log("Total False Letter Color Count: " + str(false_letter_color_count))
        Logger.log("Percentage of Successfully Detected : " + str(letter_color_detection_percentage) + "%")
        Logger.log("--------------------------------------------------")
<commit_msg>Improved ImageProcessingTester output<commit_after>import os
import json
import timeit
from ..integrated_image_processing import IntegratedImageProcessing
from ..Classifiers import *
from .settings import Settings
from .logger import Logger

class IntegratedImageProcessingTester(object):

    @staticmethod
    def complete_integrated_image_processing():
        if (os.path.isdir(Settings.IMAGE_PROCESSING_REPORT_SAVE_PATH)):
            raise Exception("Cannot create Image Processing Report Save Path: Save directory already exists")
        os.mkdir(Settings.IMAGE_PROCESSING_REPORT_SAVE_PATH)

        if (os.path.isdir(Settings.IMAGE_SAVE_PATH)):
            raise Exception("Cannot create Image Save Path: Save directory already exists")
        os.mkdir(Settings.IMAGE_SAVE_PATH)

        if (os.path.isdir(Settings.JSON_SAVE_PATH)):
            raise Exception("Cannot create Json Save Path: Save directory already exists")
        os.mkdir(Settings.JSON_SAVE_PATH)

        IntegratedImageProcessing.run_integrated_image_processing(Settings.TARGET_MAP_PATH, Settings.IMAGE_SAVE_PATH, Settings.JSON_SAVE_PATH)

    @staticmethod
    def run_integrated_image_processing_tester():
        """
        Compare the image processing result with the answers of the target map
        and print out the results.
        """
        number_of_target_maps = len(os.listdir(Settings.JSON_SAVE_PATH))

        overall_true_positive_count = 0
        overall_false_positive_count = 0

        true_shape_count = 0
        false_shape_count = 0

        true_shape_color_count = 0
        false_shape_color_count = 0

        true_letter_color_count = 0
        false_letter_color_count = 0

        overall_target_count = 0

        for index_0 in range(number_of_target_maps):

            answer_sheet = json.load(open(os.path.join(Settings.TARGET_MAP_ANSWER_SHEET_PATH, str(index_0 + 1) + ".json")))
            answer_list = []

            for index_1 in range(len(answer_sheet["targets"])):
                answer_list.append((answer_sheet["targets"][index_1]))

            overall_target_count += len(answer_list)

            target_detection_result = json.load(open(os.path.join(Settings.JSON_SAVE_PATH, str(index_0 + 1) + ".jpg.json")))
            result_list = []

            for index_2 in range(len(target_detection_result["image_processing_results"])):
                result_list.append((target_detection_result["image_processing_results"][index_2]))

            current_true_positive_count = 0
            current_false_positive_count = 0

            banned_index_list = []

            for index_3 in range(len(answer_list)):
                true_positive_found = False

                for index_4 in range(len(result_list)):
                    is_index_4_banned = False

                    for index_5 in range(len(banned_index_list)):
                        if (index_4 == banned_index_list[index_5]):
                            is_index_4_banned = True

                    if is_index_4_banned == True:
                        continue

                    correct_target_center_x = answer_list[index_3]["target_center_coordinates"][0]
                    correct_target_center_y = answer_list[index_3]["target_center_coordinates"][1]

                    detected_target_center_x = result_list[index_4]["target_location"][0]
                    detected_target_center_y = result_list[index_4]["target_location"][1]

                    if (abs(correct_target_center_x - detected_target_center_x) <= 20) and (abs(correct_target_center_y - detected_target_center_y) <= 20):
                        current_true_positive_count += 1
                        banned_index_list.append(index_4)
                        true_positive_found = True

                        current_target_shape = answer_list[index_3]["shape_type"]
                        detected_target_shape = result_list[index_4]["target_shape_type"]

                        current_target_shape_color = ColorClassifier().convert_rgba_to_color_name(answer_list[index_3]["shape_color"])
                        detected_target_shape_color = result_list[index_4]["target_shape_color"]

                        current_target_letter_color = ColorClassifier().convert_rgba_to_color_name(answer_list[index_3]["alphanumeric_color"])
                        detected_target_letter_color = result_list[index_4]["target_letter_color"]

                        if current_target_shape == detected_target_shape:
                            true_shape_count += 1
                        else:
                            false_shape_count += 1

                        if current_target_shape_color == detected_target_shape_color:
                            true_shape_color_count += 1
                        else:
                            false_shape_color_count += 1

                        if current_target_letter_color == detected_target_letter_color:
                            true_letter_color_count += 1
                        else:
                            false_letter_color_count += 1

                        continue

            current_false_positive_count = len(result_list) - current_true_positive_count

            overall_true_positive_count += current_true_positive_count
            overall_false_positive_count += current_false_positive_count

        target_detection_percentage = 100 * float(overall_true_positive_count) / (overall_target_count)
        overall_shape_detection_percentage = 100 * float(true_shape_count) / (overall_target_count)
        target_detection_based_shape_detection_percentage = 100 * float(true_shape_count) / (overall_true_positive_count)
        overall_shape_color_detection_percentage = 100 * float(true_shape_color_count) / (overall_target_count)
        target_detection_based_shape_color_detection_percentage = 100 * float(true_shape_color_count) / (overall_true_positive_count)
        overall_letter_color_detection_percentage = 100 * float(true_letter_color_count) / (overall_target_count)
        target_detection_based_letter_color_detection_percentage = 100 * float(true_letter_color_count) / (overall_true_positive_count)

        Logger.log("--------------------------------------------------")
        Logger.log("Total True Positive Count: " + str(overall_true_positive_count))
        Logger.log("Total False Positive Count: " + str(overall_false_positive_count))
        Logger.log("Percentage of Successfully Detected Targets: " + str(target_detection_percentage) + "%")
        Logger.log("--------------------------------------------------")
        Logger.log("Total True Shape Count: " + str(true_shape_count))
        Logger.log("Total False Shape Count: " + str(false_shape_count))
        Logger.log("Overall Percentage of Successfully Detected Shape: " + str(overall_shape_detection_percentage) + "%")
        Logger.log("Percentage of Successfully Detected Shape based on Target Detection Result: " + str(target_detection_based_shape_detection_percentage) + "%")
        Logger.log("--------------------------------------------------")
        Logger.log("Total True Shape Color Count: " + str(true_shape_color_count))
        Logger.log("Total False Shape Color Count: " + str(false_shape_color_count))
        Logger.log("Overall Percentage of Successfully Detected Shape Color: " + str(overall_shape_color_detection_percentage) + "%")
        Logger.log("Percentage of Successfully Detected Shape Color based on Target Detection Result: " + str(target_detection_based_shape_color_detection_percentage) + "%")
        Logger.log("--------------------------------------------------")
        Logger.log("Total True Letter Color Count: " + str(true_letter_color_count))
        Logger.log("Total False Letter Color Count: " + str(false_letter_color_count))
        Logger.log("Overall Percentage of Successfully Detected Letter Color: " + str(overall_letter_color_detection_percentage) + "%")
        Logger.log("Percentage of Successfully Detected Letter Color based on Target Detection Result: " + str(target_detection_based_letter_color_detection_percentage) + "%")
        Logger.log("--------------------------------------------------")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import rospy
import rospkg
import random
import time

from gazebo_msgs.srv import (
    SpawnModel,
    DeleteModel,
)
from geometry_msgs.msg import (
    Pose,
    Point,
)

def spawn_gazebo_models(name="candy",
                        model_type="ball",
                        model_pose=Pose(position=Point(x=0.6725, y=0.1265, z=0.7825)),
                        model_reference_frame="world"):
    # Get Models' Path
    model_path = rospkg.RosPack().get_path('turtle_war')+"/models/"

    # Load model URDF
    model_xml = ''
    with open (model_path + model_type +".urdf", "r") as ball_file:
        model_xml=ball_file.read().replace('\n', '')

    # Spawn ball URDF
    rospy.wait_for_service('/gazebo/spawn_urdf_model')
    try:
        spawn_urdf = rospy.ServiceProxy('/gazebo/spawn_urdf_model', SpawnModel)
        resp_urdf = spawn_urdf(name, model_xml, "/",
                               model_pose, model_reference_frame)
        return True

    except rospy.ServiceException, e:
        rospy.logerr("Spawn URDF service call failed: {0}".format(e))
        return False

def delete_gazebo_models(name="candy"):
    try:
        delete_model = rospy.ServiceProxy('/gazebo/delete_model', DeleteModel)
        resp_delete = delete_model(name)
    except rospy.ServiceException, e:
        rospy.loginfo("Delete Model service call failed: {0}".format(e))

def update(object_dict):

    DELETE_TIME = 5

    current_time = time.time()
    for key in list(object_dict.keys()):
        if current_time - object_dict[key] > DELETE_TIME:
            delete_gazebo_models(key)
            del object_dict[key]

def all_delete(object_dict):
    for key in list(object_dict.keys()):
        delete_gazebo_models(key)
        del object_dict[key]

def make_candy():
    r = rospy.Rate(0.25) # sampling time

    MIN_X = -1
    MAX_X = 1

    MIN_Y = -1
    MAX_Y = 1

    current_candy_num = 0
    candy_id = 0
    candy_dict ={}
    start_time = time.time()
    while time.time() - start_time < 20:
        _x = random.uniform(MIN_X,MAX_X)
        _y= random.uniform(MIN_Y,MAX_Y)
        spawn_candy_name = "candy_" + str(candy_id)
        result = spawn_gazebo_models(name=spawn_candy_name, model_type="ball",model_pose=Pose(position=Point(x=_x, y=_y)))
        if result:
            t = time.time()
            candy_dict[spawn_candy_name] = t
            candy_id +=1

        update(candy_dict)

        r.sleep()

    all_delete(candy_dict)

if __name__ == '__main__':
    rospy.init_node("make_candy_node")

    make_candy()

<commit_msg>add TIME value<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import rospy
import rospkg
import random
import time

from gazebo_msgs.srv import (
    SpawnModel,
    DeleteModel,
)
from geometry_msgs.msg import (
    Pose,
    Point,
)

TIME = 200
def spawn_gazebo_models(name="candy",
                        model_type="ball",
                        model_pose=Pose(position=Point(x=0.6725, y=0.1265, z=0.7825)),
                        model_reference_frame="world"):
    # Get Models' Path
    model_path = rospkg.RosPack().get_path('turtle_war')+"/models/"

    # Load model URDF
    model_xml = ''
    with open (model_path + model_type +".urdf", "r") as ball_file:
        model_xml=ball_file.read().replace('\n', '')

    # Spawn ball URDF
    rospy.wait_for_service('/gazebo/spawn_urdf_model')
    try:
        spawn_urdf = rospy.ServiceProxy('/gazebo/spawn_urdf_model', SpawnModel)
        resp_urdf = spawn_urdf(name, model_xml, "/",
                               model_pose, model_reference_frame)
        return True

    except rospy.ServiceException, e:
        rospy.logerr("Spawn URDF service call failed: {0}".format(e))
        return False

def delete_gazebo_models(name="candy"):
    try:
        delete_model = rospy.ServiceProxy('/gazebo/delete_model', DeleteModel)
        resp_delete = delete_model(name)
    except rospy.ServiceException, e:
        rospy.loginfo("Delete Model service call failed: {0}".format(e))

def update(object_dict):

    DELETE_TIME = 5

    current_time = time.time()
    for key in list(object_dict.keys()):
        if current_time - object_dict[key] > DELETE_TIME:
            delete_gazebo_models(key)
            del object_dict[key]

def all_delete(object_dict):
    for key in list(object_dict.keys()):
        delete_gazebo_models(key)
        del object_dict[key]

def make_candy():
    r = rospy.Rate(0.25) # sampling time

    MIN_X = -1
    MAX_X = 1

    MIN_Y = -1
    MAX_Y = 1

    current_candy_num = 0
    candy_id = 0
    candy_dict ={}
    start_time = time.time()
    while time.time() - start_time < TIME:
        _x = random.uniform(MIN_X,MAX_X)
        _y= random.uniform(MIN_Y,MAX_Y)
        spawn_candy_name = "candy_" + str(candy_id)
        result = spawn_gazebo_models(name=spawn_candy_name, model_type="ball",model_pose=Pose(position=Point(x=_x, y=_y)))
        if result:
            t = time.time()
            candy_dict[spawn_candy_name] = t
            candy_id +=1

        update(candy_dict)

        r.sleep()

    all_delete(candy_dict)

if __name__ == '__main__':
    rospy.init_node("make_candy_node")

    make_candy()

<|endoftext|>


<commit_before>'''
Created on Jun 21, 2012

@author: dstrauss
'''

import xml.etree.ElementTree as xml
import subprocess
import time
import sys

def waitForExit(jobName):
    ''' Routine to check and see if a particular job is still running, if not, return '''
    doExit = False
    while doExit == False:
        pipeOut = subprocess.Popen(['qstat', '-xml', jobName], stdout=subprocess.PIPE)
        f = pipeOut.stdout.read()
        P = xml.fromstring(f)
        for z in P.getiterator():
            if z.tag == 'job_state':
                if z.text == 'R':
                    pass
                    # print 'Still Running ' + jobName
                elif z.text == 'C':
                    doExit = True
                    print 'Finished ' + jobName

        time.sleep(5)

def submitJob(cmd):
    pipeOut = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    f = pipeOut.stdout.read()
    f = f[:-1]
    print f + ' ' + time.asctime(time.localtime())
    # time.sleep(1)
    return f

def main():
    for ix in range(6,200):
        jobTitle = 'run' + sys.argv[1] + repr(ix)
        fileName = 'sub' + sys.argv[1] + repr(ix) + '.pbs'
        fid = open(fileName, 'w')
        fid.write('mpiexec -wdir /home/sgeadmin/subsurface/src/ python coordinate.py ' + sys.argv[1] + ' ' + repr(ix) + ' ' + 'TE')
        fid.close()
        cmd = ['qsub', '-pe', 'orte', '16', '-cwd', fileName]
        print cmd
        ppid = submitJob(cmd)
        print ppid
        # waitForExit(ppid)

if __name__=='__main__':
    main()
<commit_msg>small chngs.<commit_after>'''
Created on Jun 21, 2012

@author: dstrauss
'''

import xml.etree.ElementTree as xml
import subprocess
import time
import sys

def waitForExit(jobName):
    ''' Routine to check and see if a particular job is still running, if not, return '''
    doExit = False
    while doExit == False:
        pipeOut = subprocess.Popen(['qstat', '-xml', jobName], stdout=subprocess.PIPE)
        f = pipeOut.stdout.read()
        P = xml.fromstring(f)
        for z in P.getiterator():
            if z.tag == 'job_state':
                if z.text == 'R':
                    pass
                    # print 'Still Running ' + jobName
                elif z.text == 'C':
                    doExit = True
                    print 'Finished ' + jobName

        time.sleep(5)

def submitJob(cmd):
    pipeOut = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    f = pipeOut.stdout.read()
    f = f[:-1]
    print f + ' ' + time.asctime(time.localtime())
    # time.sleep(1)
    return f

def main():
    for ix in range(1):
        jobTitle = 'run' + sys.argv[1] + repr(ix)
        fileName = 'sub' + sys.argv[1] + repr(ix) + '.pbs'
        fid = open(fileName, 'w')
        fid.write('mpiexec -wdir /home/sgeadmin/subsurface/src/ python awsDummy.py ') # + sys.argv[1] + ' ' + repr(ix) + ' ' + 'TE')
        fid.close()
        cmd = ['qsub', '-pe', 'orte', '1', '-cwd', fileName]
        print cmd
        ppid = submitJob(cmd)
        print ppid

        # waitForExit(ppid)

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2014, 2015, 2016, 2017 Timothy Savannah under LGPL version 2.1. See LICENSE for more information.
#
# fields - Some types and objects related to advanced fields
#

# vim:set ts=8 shiftwidth=8 softtabstop=8 noexpandtab :

__all__ = ('IRField', 'IRNullType', 'irNull', 'IRPickleField',
	'IRCompressedField', 'IRUnicodeField', 'IRRawField', 'IRBase64Field',
	'IRFixedPointField', 'IRDatetimeValue', 'IRJsonValue',
	'IRBytesField', 'IRClassicField',
	'IR_NULL_STR', 'IR_NULL_BYTES', 'IR_NULL_UNICODE', 'IR_NULL_STRINGS' )

import sys
from datetime import datetime

from ..compat_str import to_unicode, tobytes
from ..deprecated import deprecatedMessage, deprecated

from .null import irNull, IR_NULL_STRINGS, IR_NULL_STR, IR_NULL_BYTES, IR_NULL_UNICODE, IRNullType

from hashlib import md5

try:
	unicode
except NameError:
	unicode = str

class IRField(str):
	'''
		IRField - An advanced field

		If a type is defined other than default/str/bytes/None , an empty value (empty string in Redis) will be assigned to the IRNullType instance provided in this module, irNull.
		irNull does not equal anything except irNull (or another IRNullType). Use this to check if a value has been assigned for other types.

		BE VERY CAREFUL ABOUT USING "float" as a type! It is an inprecise field and can vary from system to system. Instead of using a float,
		consider using fields.IRFixedPointField, which is indexable.

		An IRField may be indexable (depending on the type), and has the option to hash the index

	'''

	# CAN_INDEX - Set this to True if this type can be indexed. Otherwise, set it to False to disallow indexing on this field.
        #    The object itself is checked, so if a field is generally indexable except under certain conditions, the class can have
        #      True while the specific object that should be disallowed can be False.
        #
        # If IRField base class is used, the following types are CAN_INDEX=True: str, unicode, int, bool.  Otherwise, if CAN_INDEX is defined
        #   on the type, that value will be used.
	CAN_INDEX = False

	# Start as a class variable, so "toIndex" works even if IRField constructor is not called (which really shouldn't be called on extending classes)
	hashIndex = False

	# The default value of this class
	defaultValue = irNull

	def __init__(self, name='', valueType=str, defaultValue=irNull, hashIndex=False):
		'''
			__init__ - Create an IRField. Use this directly in the FIELDS array for advanced functionality on a field.

			@param name <str> - The name of this field

			@param valueType <type> - The type that will be used for this field. Default str/unicode (and bytes on python2)
				act the same as non-IRField FIELDS entries (just plain string), i.e. they are encoded to unicode to and from Redis.

				If you pass in None, then no decoding will take place (so whatever you provide goes in, and bytes come out of Redis).
				This is similar to IRFieldRaw

				On python3, if you pass bytes, than the field will be left as bytes.
				To be both python2 and python3 compatible, however, you can use IRBytesField

				If bool is used, then "1" and "true" are True, "0" and "false" are False, any other value is an exception.

				You can also pass an advanced type (see IndexedRedis.fields.FieldValueTypes) like datetime and json.

				All types other than string/unicode/bytes/None will be assigned 'irNull' if they were not provided a value.
				@see irNull - Equals only irNull (or other IRNullType instances). You can use this to check if an integer is defined versus 0, for example.

				While this class is create for primitive types (like int's and datetimes), more complex types extend IRField (such as pickle, compressed, or unicode with a specific encoding).

			@param defaultValue <any> (default irNull) - The value assigned to this field as a "default", i.e. when no value has yet been set. Generally, it makes sense to keep this as irNull, but you may want a different default.

			@param hashIndex <bool> (default False) - If true, the md5 hash of the value will be used for indexing and filtering. This may be useful for very long fields.

			An IRField may be indexable (depending on the type), and has the option to hash the index

			NOTE: If you are extending IRField, you should probably not call this __init__ function. So long as you implement your own "convert", any fields used are set on a class-level.
		'''
		self.defaultValue = defaultValue
		self.hashIndex = hashIndex

		if valueType in (str, unicode):
			valueType = str
			self._fromStorage = self._convertStr
			self._fromInput = self._convertStr
			self._toStorage = self._convertStr
		elif bytes != str and valueType == bytes:
			valueType = bytes
			self._fromStorage = self._convertBytes
			self._fromInput = self._convertBytes
			self._toStorage = self._convertBytes

			# Cannot index here, but CAN index if using IRBytesField. This is because python2 and python3 could handle it differently in certain cases.
			self.CAN_INDEX = False
		elif valueType in (None, type(None)):
			self._fromStorage = self._noConvert
			self._fromInput = self._noConvert
			self._toStorage = self._noConvert
			self.CAN_INDEX = False
		# I don't like these next two conditions, but it will train folks to use the correct types (whereas they may just try to shove dict in, and give up that it doesn't work)
		elif valueType in (dict, list, tuple):
#			deprecatedMessage('WARNING: Implicitly converting IRField(%s, valueType=%s) to IRField(%s, valueType=IndexedRedis.fields.FieldValueTypes.IRJsonValue)\n' %(repr(name), valueType.__name__, repr(name)), printStack=True)
			valueType = IRJsonValue
			self.CAN_INDEX = IRJsonValue.CAN_INDEX
		elif valueType == datetime:
#			deprecatedMessage('WARNING: Implicitly converting IRField(%s, valueType=datetime.datetime) to IRField(%s, valueType=IndexedRedis.fields.FieldValueTypes.IRDatetimeValue)\n' %(repr(name), repr(name)), printStack=True)
			valueType = IRDatetimeValue
			self.CAN_INDEX = IRDatetimeValue.CAN_INDEX
		else:
			if not isinstance(valueType, type):
				raise TypeError('valueType %s is not a type. Use int, str, etc' %(repr(valueType,)))
			if valueType == bool:
				self._fromStorage = self._convertBool
				self._fromInput = self._convertBool
			elif isinstance(valueType, (set, frozenset, )):
				raise TypeError('set types are not supported types. Use IRPickleField to store pickles of any type (which allow storing objects, etc), or use IRField(.. valueType=IRJsonValue) to store basic data (strings, integers) in lists.')
		self.valueType = valueType

		if valueType in (str, unicode, int, bool):
			self.CAN_INDEX = True
		elif hasattr(valueType, 'CAN_INDEX'):
			self.CAN_INDEX = valueType.CAN_INDEX
		# XXX: Commented because default CAN_INDEX is False.
#		elif valueType == float:
#			# Floats are not filterable/indexable across platforms, as they may have different rounding issues, or different number
#			#  of points of accuracy, etc.
#			# Use fields.IRFixedPointField if you need to index/filter on a floating point value.
#			self.CAN_INDEX = False
#		elif issubclass(valueType.__class__, object):
#			# Don't allow objects to index by default unless they define CAN_INDEX to be True
#			self.CAN_INDEX = False

	def toStorage(self, value):
		'''
			toStorage - Convert the value to a string representation for storage.

			  The default implementation will work here for basic types.

			@param value - The value of the item to convert
			@return A string value suitable for storing.
		'''
		if value == irNull or None:
			return IR_NULL_STR

		return self._toStorage(value)

	def _toStorage(self, value):
		'''
			_toStorage - Convert the value to a string for storage.

			The default implementation works for most valueTypes within IRField, override this for extending types.

			You don't need to handle null

			@param value - Value of item to convert

			@return - A string value suitable for storing
		'''
		return to_unicode(value)

	def fromStorage(self, value):
		'''
			fromStorage - Convert the value from storage to the value type.

			@param value - Value to convert

			@return - The converted value
		'''
		if value in IR_NULL_STRINGS:
			return irNull

		return self._fromStorage(value)

	def _fromStorage(self, value):
		'''
			_fromStorage - Convert the value from storage to the value type.

			  This default impl works fine for most value types, should be implemented by extending types.

			  @param value - Value to convert

			  @return - Converted value
		'''
		return self.valueType(value)

	@deprecated("convert is deprecated. Please use fromStorage instead. See fromStorage docstring if extending.")
	def convert(self, value):
		return self.fromStorage(value)

	def fromInput(self, value):
		'''
			fromInput - Convert the value from input (like assigning this through constructor or as an item assignment on the object

			@param value - Value to convert

			@return - Converted value
		'''
		if value == irNull:
			return irNull

		return self._fromInput(value)

	def _fromInput(self, value):
		'''
			_fromInput - Convert the value from input. Implement this in extending types.

			@param value - Value to convert

			@return converted value
		'''
		return self.valueType(value)

	'''
		_toIndex - Prepare a value for index.

		  Does not need to handle nulls or hasing.

		  You probably won't need to implement this, but IRFieldChain for example does.
	'''
	_toIndex = toStorage

	def toIndex(self, value):
		'''
			toIndex - An optional method which will return the value prepped for index.

			By default, "toStorage" will be called. If you provide "hashIndex=True" on the constructor,
			the field will be md5summed for indexing purposes. This is useful for large strings, etc.
		'''
		if self._isIrNull(value):
			ret = IR_NULL_STR
		else:
			ret = self._toIndex(value)

		if self.isIndexHashed is False:
			return ret

		return md5(tobytes(ret)).hexdigest()

	def getDefaultValue(self):
		return self.defaultValue

	@property
	def isIndexHashed(self):
		'''
			isIndexHashed - Returns if the index value should be hashed

			@return <bool> - True if this field should be hashed before indexing / filtering
		'''
		return bool(self.hashIndex)

	@property
	def name(self):
		'''
			name - Property, return this field's name

			@return <str> - Field name
		'''
		return str(self)

	def _convertStr(self, value):
		return to_unicode(value)

	def _convertBytes(self, value):
		return tobytes(value)

	def _noConvert(self, value):
		return value

	def _convertBool(self, value):
		if self._isNullValue(value):
			return irNull
		xvalue = value.lower()
		if xvalue in ('true', '1'):
			return True
		elif xvalue in ('false', '0'):
			return False

		# I'm not sure what to do here... Should we raise an exception because the data is invalid? Should just return True?
		raise ValueError('Unexpected value for bool type: %s' %(value,))

	@staticmethod
	def _isNullValue(value):
		'''
			_isNullValue - Tests value if it should be represented by irNull.

			convert and toStorage should test if value is null and return null (for most types)
		'''
		return bool(value in (b'', '', irNull, None) or value in IR_NULL_STRINGS )

	@staticmethod
	def _isIrNull(value):
		return bool( value == irNull or value in IR_NULL_STRINGS )

	def _getReprProperties(self):
		'''
			_getReprProperties - Get the properties of this field to display in repr().

				These should be in the form of $propertyName=$propertyRepr

				The default IRField implementation handles just the "hashIndex" property.

				defaultValue is part of "__repr__" impl. You should just extend this method
				with your object's properties instead of rewriting repr.

		'''
		ret = []
		if getattr(self, 'valueType', None) is not None:
			ret.append('valueType=%s' %(self.valueType.__name__, ))
		if hasattr(self, 'hashIndex'):
			ret.append('hashIndex=%s' %(self.hashIndex, ))

		return ret

	def __repr__(self):
		'''
			__repr__ - Return an object-representation string of this field instance.

			You should NOT need to extend this on your IRField, instead just implement _getReprProperties

			  to return your type's specific properties associated with this instance.

			  @see _getReprProperties
		'''
		ret = [ self.__class__.__name__, '( ', '"%s"' %(str(self), ) ]

		reprProperties = self._getReprProperties()

		defaultValue = self.getDefaultValue()
		if defaultValue == irNull:
			reprProperties.append('defaultValue=irNull')
		else:
			reprProperties.append('defaultValue=%s' %(repr(defaultValue), ))

		ret.append(', ')
		ret.append(', '.join(reprProperties))

		ret.append(' )')

		return ''.join(ret)

	def copy(self):
		'''
			copy - Create a copy of this IRField.

			  Each subclass should implement this, as you'll need to pass in the args to constructor.

			@return <IRField (or subclass)> - Another IRField that has all the same values as this one.
		'''
		return self.__class__(name=self.name, valueType=self.valueType, defaultValue=self.defaultValue, hashIndex=self.hashIndex)

	def __new__(self, name='', valueType=None, defaultValue=irNull, hashIndex=False):
		return str.__new__(self, name)

from .compressed import IRCompressedField
from .pickle_field import IRPickleField
from .unicode_field import IRUnicodeField
from .raw import IRRawField
from .chain import IRFieldChain
from .b64 import IRBase64Field
from .fixedpoint import IRFixedPointField
from .bytes_field import IRBytesField
from .foreign import IRForeignLinkField, IRForeignLinkFieldBase, IRForeignMultiLinkField

from .classic import IRClassicField

from .FieldValueTypes import IRDatetimeValue, IRJsonValue

# vim:set ts=8 shiftwidth=8 softtabstop=8 noexpandtab :
<commit_msg>Include new foreign fields in __all__<commit_after># Copyright (c) 2014, 2015, 2016, 2017 Timothy Savannah under LGPL version 2.1. See LICENSE for more information.
#
# fields - Some types and objects related to advanced fields
#

# vim:set ts=8 shiftwidth=8 softtabstop=8 noexpandtab :

__all__ = ('IRField', 'IRNullType', 'irNull', 'IRPickleField',
	'IRCompressedField', 'IRUnicodeField', 'IRRawField', 'IRBase64Field',
	'IRFixedPointField', 'IRDatetimeValue', 'IRJsonValue',
	'IRBytesField', 'IRClassicField',
	'IRForeignLinkFieldBase', 'IRForeignLinkField', 'IRForeignMultiLinkField',
	'IR_NULL_STR', 'IR_NULL_BYTES', 'IR_NULL_UNICODE', 'IR_NULL_STRINGS' )

import sys
from datetime import datetime

from ..compat_str import to_unicode, tobytes
from ..deprecated import deprecatedMessage, deprecated

from .null import irNull, IR_NULL_STRINGS, IR_NULL_STR, IR_NULL_BYTES, IR_NULL_UNICODE, IRNullType

from hashlib import md5

try:
	unicode
except NameError:
	unicode = str

class IRField(str):
	'''
		IRField - An advanced field

		If a type is defined other than default/str/bytes/None , an empty value (empty string in Redis) will be assigned to the IRNullType instance provided in this module, irNull.
		irNull does not equal anything except irNull (or another IRNullType). Use this to check if a value has been assigned for other types.

		BE VERY CAREFUL ABOUT USING "float" as a type! It is an inprecise field and can vary from system to system. Instead of using a float,
		consider using fields.IRFixedPointField, which is indexable.

		An IRField may be indexable (depending on the type), and has the option to hash the index

	'''

	# CAN_INDEX - Set this to True if this type can be indexed. Otherwise, set it to False to disallow indexing on this field.
        #    The object itself is checked, so if a field is generally indexable except under certain conditions, the class can have
        #      True while the specific object that should be disallowed can be False.
        #
        # If IRField base class is used, the following types are CAN_INDEX=True: str, unicode, int, bool.  Otherwise, if CAN_INDEX is defined
        #   on the type, that value will be used.
	CAN_INDEX = False

	# Start as a class variable, so "toIndex" works even if IRField constructor is not called (which really shouldn't be called on extending classes)
	hashIndex = False

	# The default value of this class
	defaultValue = irNull

	def __init__(self, name='', valueType=str, defaultValue=irNull, hashIndex=False):
		'''
			__init__ - Create an IRField. Use this directly in the FIELDS array for advanced functionality on a field.

			@param name <str> - The name of this field

			@param valueType <type> - The type that will be used for this field. Default str/unicode (and bytes on python2)
				act the same as non-IRField FIELDS entries (just plain string), i.e. they are encoded to unicode to and from Redis.

				If you pass in None, then no decoding will take place (so whatever you provide goes in, and bytes come out of Redis).
				This is similar to IRFieldRaw

				On python3, if you pass bytes, than the field will be left as bytes.
				To be both python2 and python3 compatible, however, you can use IRBytesField

				If bool is used, then "1" and "true" are True, "0" and "false" are False, any other value is an exception.

				You can also pass an advanced type (see IndexedRedis.fields.FieldValueTypes) like datetime and json.

				All types other than string/unicode/bytes/None will be assigned 'irNull' if they were not provided a value.
				@see irNull - Equals only irNull (or other IRNullType instances). You can use this to check if an integer is defined versus 0, for example.

				While this class is create for primitive types (like int's and datetimes), more complex types extend IRField (such as pickle, compressed, or unicode with a specific encoding).

			@param defaultValue <any> (default irNull) - The value assigned to this field as a "default", i.e. when no value has yet been set. Generally, it makes sense to keep this as irNull, but you may want a different default.

			@param hashIndex <bool> (default False) - If true, the md5 hash of the value will be used for indexing and filtering. This may be useful for very long fields.

			An IRField may be indexable (depending on the type), and has the option to hash the index

			NOTE: If you are extending IRField, you should probably not call this __init__ function. So long as you implement your own "convert", any fields used are set on a class-level.
		'''
		self.defaultValue = defaultValue
		self.hashIndex = hashIndex

		if valueType in (str, unicode):
			valueType = str
			self._fromStorage = self._convertStr
			self._fromInput = self._convertStr
			self._toStorage = self._convertStr
		elif bytes != str and valueType == bytes:
			valueType = bytes
			self._fromStorage = self._convertBytes
			self._fromInput = self._convertBytes
			self._toStorage = self._convertBytes

			# Cannot index here, but CAN index if using IRBytesField. This is because python2 and python3 could handle it differently in certain cases.
			self.CAN_INDEX = False
		elif valueType in (None, type(None)):
			self._fromStorage = self._noConvert
			self._fromInput = self._noConvert
			self._toStorage = self._noConvert
			self.CAN_INDEX = False
		# I don't like these next two conditions, but it will train folks to use the correct types (whereas they may just try to shove dict in, and give up that it doesn't work)
		elif valueType in (dict, list, tuple):
#			deprecatedMessage('WARNING: Implicitly converting IRField(%s, valueType=%s) to IRField(%s, valueType=IndexedRedis.fields.FieldValueTypes.IRJsonValue)\n' %(repr(name), valueType.__name__, repr(name)), printStack=True)
			valueType = IRJsonValue
			self.CAN_INDEX = IRJsonValue.CAN_INDEX
		elif valueType == datetime:
#			deprecatedMessage('WARNING: Implicitly converting IRField(%s, valueType=datetime.datetime) to IRField(%s, valueType=IndexedRedis.fields.FieldValueTypes.IRDatetimeValue)\n' %(repr(name), repr(name)), printStack=True)
			valueType = IRDatetimeValue
			self.CAN_INDEX = IRDatetimeValue.CAN_INDEX
		else:
			if not isinstance(valueType, type):
				raise TypeError('valueType %s is not a type. Use int, str, etc' %(repr(valueType,)))
			if valueType == bool:
				self._fromStorage = self._convertBool
				self._fromInput = self._convertBool
			elif isinstance(valueType, (set, frozenset, )):
				raise TypeError('set types are not supported types. Use IRPickleField to store pickles of any type (which allow storing objects, etc), or use IRField(.. valueType=IRJsonValue) to store basic data (strings, integers) in lists.')
		self.valueType = valueType

		if valueType in (str, unicode, int, bool):
			self.CAN_INDEX = True
		elif hasattr(valueType, 'CAN_INDEX'):
			self.CAN_INDEX = valueType.CAN_INDEX
		# XXX: Commented because default CAN_INDEX is False.
#		elif valueType == float:
#			# Floats are not filterable/indexable across platforms, as they may have different rounding issues, or different number
#			#  of points of accuracy, etc.
#			# Use fields.IRFixedPointField if you need to index/filter on a floating point value.
#			self.CAN_INDEX = False
#		elif issubclass(valueType.__class__, object):
#			# Don't allow objects to index by default unless they define CAN_INDEX to be True
#			self.CAN_INDEX = False

	def toStorage(self, value):
		'''
			toStorage - Convert the value to a string representation for storage.

			  The default implementation will work here for basic types.

			@param value - The value of the item to convert
			@return A string value suitable for storing.
		'''
		if value == irNull or None:
			return IR_NULL_STR

		return self._toStorage(value)

	def _toStorage(self, value):
		'''
			_toStorage - Convert the value to a string for storage.

			The default implementation works for most valueTypes within IRField, override this for extending types.

			You don't need to handle null

			@param value - Value of item to convert

			@return - A string value suitable for storing
		'''
		return to_unicode(value)

	def fromStorage(self, value):
		'''
			fromStorage - Convert the value from storage to the value type.

			@param value - Value to convert

			@return - The converted value
		'''
		if value in IR_NULL_STRINGS:
			return irNull

		return self._fromStorage(value)

	def _fromStorage(self, value):
		'''
			_fromStorage - Convert the value from storage to the value type.

			  This default impl works fine for most value types, should be implemented by extending types.

			  @param value - Value to convert

			  @return - Converted value
		'''
		return self.valueType(value)

	@deprecated("convert is deprecated. Please use fromStorage instead. See fromStorage docstring if extending.")
	def convert(self, value):
		return self.fromStorage(value)

	def fromInput(self, value):
		'''
			fromInput - Convert the value from input (like assigning this through constructor or as an item assignment on the object

			@param value - Value to convert

			@return - Converted value
		'''
		if value == irNull:
			return irNull

		return self._fromInput(value)

	def _fromInput(self, value):
		'''
			_fromInput - Convert the value from input. Implement this in extending types.

			@param value - Value to convert

			@return converted value
		'''
		return self.valueType(value)

	'''
		_toIndex - Prepare a value for index.

		  Does not need to handle nulls or hashing.

		  You probably won't need to implement this, but IRFieldChain for example does.
	'''
	_toIndex = toStorage

	def toIndex(self, value):
		'''
			toIndex - An optional method which will return the value prepped for index.

			By default, "toStorage" will be called. If you provide "hashIndex=True" on the constructor,
			the field will be md5summed for indexing purposes. This is useful for large strings, etc.
		'''
		if self._isIrNull(value):
			ret = IR_NULL_STR
		else:
			ret = self._toIndex(value)

		if self.isIndexHashed is False:
			return ret

		return md5(tobytes(ret)).hexdigest()

	def getDefaultValue(self):
		return self.defaultValue

	@property
	def isIndexHashed(self):
		'''
			isIndexHashed - Returns if the index value should be hashed

			@return <bool> - True if this field should be hashed before indexing / filtering
		'''
		return bool(self.hashIndex)

	@property
	def name(self):
		'''
			name - Property, return this field's name

			@return <str> - Field name
		'''
		return str(self)

	def _convertStr(self, value):
		return to_unicode(value)

	def _convertBytes(self, value):
		return tobytes(value)

	def _noConvert(self, value):
		return value

	def _convertBool(self, value):
		if self._isNullValue(value):
			return irNull
		xvalue = value.lower()
		if xvalue in ('true', '1'):
			return True
		elif xvalue in ('false', '0'):
			return False

		# I'm not sure what to do here... Should we raise an exception because the data is invalid? Should just return True?
		raise ValueError('Unexpected value for bool type: %s' %(value,))

	@staticmethod
	def _isNullValue(value):
		'''
			_isNullValue - Tests value if it should be represented by irNull.

			convert and toStorage should test if value is null and return null (for most types)
		'''
		return bool(value in (b'', '', irNull, None) or value in IR_NULL_STRINGS )

	@staticmethod
	def _isIrNull(value):
		return bool( value == irNull or value in IR_NULL_STRINGS )

	def _getReprProperties(self):
		'''
			_getReprProperties - Get the properties of this field to display in repr().

				These should be in the form of $propertyName=$propertyRepr

				The default IRField implementation handles just the "hashIndex" property.

				defaultValue is part of "__repr__" impl. You should just extend this method
				with your object's properties instead of rewriting repr.

		'''
		ret = []
		if getattr(self, 'valueType', None) is not None:
			ret.append('valueType=%s' %(self.valueType.__name__, ))
		if hasattr(self, 'hashIndex'):
			ret.append('hashIndex=%s' %(self.hashIndex, ))

		return ret

	def __repr__(self):
		'''
			__repr__ - Return an object-representation string of this field instance.

			You should NOT need to extend this on your IRField, instead just implement _getReprProperties

			  to return your type's specific properties associated with this instance.

			  @see _getReprProperties
		'''
		ret = [ self.__class__.__name__, '( ', '"%s"' %(str(self), ) ]

		reprProperties = self._getReprProperties()

		defaultValue = self.getDefaultValue()
		if defaultValue == irNull:
			reprProperties.append('defaultValue=irNull')
		else:
			reprProperties.append('defaultValue=%s' %(repr(defaultValue), ))

		ret.append(', ')
		ret.append(', '.join(reprProperties))

		ret.append(' )')

		return ''.join(ret)

	def copy(self):
		'''
			copy - Create a copy of this IRField.

			  Each subclass should implement this, as you'll need to pass in the args to constructor.

			@return <IRField (or subclass)> - Another IRField that has all the same values as this one.
		'''
		return self.__class__(name=self.name, valueType=self.valueType, defaultValue=self.defaultValue, hashIndex=self.hashIndex)

	def __new__(self, name='', valueType=None, defaultValue=irNull, hashIndex=False):
		return str.__new__(self, name)

from .compressed import IRCompressedField
from .pickle_field import IRPickleField
from .unicode_field import IRUnicodeField
from .raw import IRRawField
from .chain import IRFieldChain
from .b64 import IRBase64Field
from .fixedpoint import IRFixedPointField
from .bytes_field import IRBytesField
from .foreign import IRForeignLinkField, IRForeignLinkFieldBase, IRForeignMultiLinkField

from .classic import IRClassicField

from .FieldValueTypes import IRDatetimeValue, IRJsonValue

# vim:set ts=8 shiftwidth=8 softtabstop=8 noexpandtab :
<|endoftext|>


<commit_before>import pytest
import os
import sys
import re

from peewee import PeeweeException
from pprint import pprint

os.environ['SENTINEL_ENV'] = 'test'
sys.path.append( os.path.join( os.path.dirname(__file__), '..', 'lib' ) )
from models import GovernanceObject, Proposal, Event, Superblock

# NGM/TODO: setup both Proposal and Superblock, and insert related rows,
# including Events

def setup():

    # clear tables first...
    Event.delete().execute()
    Proposal.delete().execute()
    Superblock.delete().execute()
    GovernanceObject.delete().execute()

def teardown():
    pass

@pytest.fixture
def superblock():
    from models import Superblock

    # NOTE: no governance_object_id is set
    sbobj = Superblock(
        name = "sb1803405",
        event_block_height = 62500,
        payment_addresses = "yYe8KwyaUu5YswSYmB3q3ryx8XTUu9y7Ui|yTC62huR4YQEPn9AJHjnQxxreHSbgAoatV",
        payment_amounts  = "5|3"
    )

    # NOTE: this object is (intentionally) not saved yet.
    #       We want to return an built, but unsaved, object
    return sbobj

@pytest.fixture
def proposal():
    from models import Proposal

    # NOTE: no governance_object_id is set
    pobj = Proposal(
        start_epoch     = 1483250400,  # 2017-01-01
        end_epoch       = 1491022800,  # 2017-04-01
        name   = "chrono-trigger-party",
        description_url = "https://dashcentral.com/chrono-trigger-party",
        payment_address = "yYe8KwyaUu5YswSYmB3q3ryx8XTUu9y7Ui",
        payment_amount  = 7
    )

    # NOTE: this object is (intentionally) not saved yet.
    #       We want to return an built, but unsaved, object
    return pobj

# GovernanceObject model
@pytest.fixture
def governance_object():
    from models import GovernanceObject
    govobj = GovernanceObject()
    # NOTE: do not save, return an unsaved govobj

    return govobj

# Starting with this info...
#
# proposal -c -p chrono-trigger-party -d https://dashcentral.com/chrono-trigger-party -s 2016/09/01 -e 2016/10/01 -x yYe8KwyaUu5YswSYmB3q3ryx8XTUu9y7Ui -a 100
# superblock -c -p yYe8KwyaUu5YswSYmB3q3ryx8XTUu9y7Ui=5,yTC62huR4YQEPn9AJHjnQxxreHSbgAoatV=3 -b 62500
#
# ... how do we get the actual "gobject" command used with dash-cli ?
#
# govobj = GovernanceObject()
# govobj.load(pw_event.governance_object_id)

def test_prepare_command(proposal):
    proposal.create_and_queue()
    gobj = proposal.governance_object

    # some small manipulations for our test cases
    gobj.object_creation_time = 1471898632
    gobj.save()

    # update primary key for our test -- can't be done w/save() method
    uq = GovernanceObject.update(id = 5).where(GovernanceObject.id == gobj.id)
    uq.execute()

    # reload stale objects
    proposal = Proposal.get( Proposal.id == proposal.id )
    gobj = GovernanceObject.get( GovernanceObject.id == 5 )

    # ensure tight regex match
    prepare_command_regex = re.compile('^gobject prepare ([\da-f]+) ([\da-f]+) ([\d]+) ([\w-]+) ([\da-f]+)$')

    cmd = gobj.get_prepare_command()

    match = prepare_command_regex.search(cmd)
    assert match != None

    # if match:
    #     # print "matched!"
    #     print(match.group(1))
    #     print(match.group(2))
    #     print(match.group(3))
    #     print(match.group(4))
    #     print(match.group(5))
    # else:
    #     print "NO MATCH!"

    gobject_command = "gobject prepare 0 1 1471898632 chrono-trigger-party 5b5b2270726f706f73616c222c207b22656e645f65706f6368223a20313439313032323830302c2022676f7665726e616e63655f6f626a6563745f6964223a20352c20227061796d656e745f61646472657373223a2022795965384b77796155753559737753596d4233713372797838585455753979375569222c20227061796d656e745f616d6f756e74223a20372e30303030303030302c202270726f706f73616c5f6e616d65223a20226368726f6e6f2d747269676765722d7061727479222c202273746172745f65706f6368223a20313438333235303430302c202274797065223a20317d5d5d"
    assert cmd == gobject_command

# ensure all 3 rows get created -- govobj, proposal, and event.
def test_proposal_create_and_queue(proposal):
    from models import GovernanceObject, Proposal, Event

    proposal_count = Proposal.select().count()
    event_count    = Event.select().count()
    gov_obj_count  = GovernanceObject.select().count()
    total_count = proposal_count + event_count + gov_obj_count

    assert total_count == 0

    try:
        proposal.create_and_queue()
    except PeeweeException as e:
        print "error: %s" % e[1]

    proposal_count = Proposal.select().count()
    event_count    = Event.select().count()
    gov_obj_count  = GovernanceObject.select().count()
    total_count = proposal_count + event_count + gov_obj_count

    assert proposal_count == 1
    assert event_count    == 1
    assert gov_obj_count  == 1
    assert total_count == 3

# ensure all 3 rows get created -- govobj, proposal, and event.
def test_superblock_create_and_queue(superblock):
    from models import GovernanceObject, Superblock, Event

    superblock_count = Superblock.select().count()
    event_count    = Event.select().count()
    gov_obj_count  = GovernanceObject.select().count()
    total_count = superblock_count + event_count + gov_obj_count

    assert total_count == 0

    try:
        superblock.create_and_queue()
    except PeeweeException as e:
        print "error: %s" % e[1]

    superblock_count = Superblock.select().count()
    event_count    = Event.select().count()
    gov_obj_count  = GovernanceObject.select().count()
    total_count = superblock_count + event_count + gov_obj_count

    assert superblock_count == 1
    assert event_count      == 1
    assert gov_obj_count    == 1
    assert total_count      == 3
<commit_msg>fix test for new DB schema changes<commit_after>import pytest
import os
import sys
import re

from peewee import PeeweeException
from pprint import pprint

os.environ['SENTINEL_ENV'] = 'test'
sys.path.append( os.path.join( os.path.dirname(__file__), '..', 'lib' ) )
from models import GovernanceObject, Proposal, Event, Superblock

# NGM/TODO: setup both Proposal and Superblock, and insert related rows,
# including Events

def setup():

    # clear tables first...
    Event.delete().execute()
    Proposal.delete().execute()
    Superblock.delete().execute()
    GovernanceObject.delete().execute()

def teardown():
    pass

@pytest.fixture
def superblock():
    from models import Superblock

    # NOTE: no governance_object_id is set
    sbobj = Superblock(
        name = "sb1803405",
        event_block_height = 62500,
        payment_addresses = "yYe8KwyaUu5YswSYmB3q3ryx8XTUu9y7Ui|yTC62huR4YQEPn9AJHjnQxxreHSbgAoatV",
        payment_amounts  = "5|3"
    )

    # NOTE: this object is (intentionally) not saved yet.
    #       We want to return an built, but unsaved, object
    return sbobj

@pytest.fixture
def proposal():
    from models import Proposal

    # NOTE: no governance_object_id is set
    pobj = Proposal(
        start_epoch     = 1483250400,  # 2017-01-01
        end_epoch       = 1491022800,  # 2017-04-01
        name   = "beer-reimbursement-7",
        description_url = "https://dashcentral.com/beer-reimbursement-7",
        payment_address = "yYe8KwyaUu5YswSYmB3q3ryx8XTUu9y7Ui",
        payment_amount  = 7
    )

    # NOTE: this object is (intentionally) not saved yet.
    #       We want to return an built, but unsaved, object
    return pobj

# GovernanceObject model
@pytest.fixture
def governance_object():
    from models import GovernanceObject
    govobj = GovernanceObject()
    # NOTE: do not save, return an unsaved govobj

    return govobj

def test_prepare_command(proposal):
    proposal.create_and_queue()
    gobj = proposal.governance_object

    # some small manipulations for our test cases
    gobj.object_creation_time = 1471898632
    gobj.save()

    # update primary key for our test -- can't be done w/save() method
    uq = GovernanceObject.update(id = 5).where(GovernanceObject.id == gobj.id)
    uq.execute()

    # reload stale objects
    proposal = Proposal.get( Proposal.id == proposal.id )
    gobj = GovernanceObject.get( GovernanceObject.id == 5 )

    # ensure tight regex match
    prepare_command_regex = re.compile('^gobject prepare ([\da-f]+) ([\da-f]+) ([\d]+) ([\w-]+) ([\da-f]+)$')

    cmd = gobj.get_prepare_command()

    match = prepare_command_regex.search(cmd)
    assert match != None

    # if match:
    #     # print "matched!"
    #     print(match.group(1))
    #     print(match.group(2))
    #     print(match.group(3))
    #     print(match.group(4))
    #     print(match.group(5))
    # else:
    #     print "NO MATCH!"

    gobject_command = "gobject prepare 0 1 1471898632 beer-reimbursement-7 5b5b2270726f706f73616c222c207b22656e645f65706f6368223a20313439313032323830302c2022676f7665726e616e63655f6f626a6563745f6964223a20352c20226e616d65223a2022626565722d7265696d62757273656d656e742d37222c20227061796d656e745f61646472657373223a2022795965384b77796155753559737753596d4233713372797838585455753979375569222c20227061796d656e745f616d6f756e74223a20372e30303030303030302c202273746172745f65706f6368223a20313438333235303430302c202274797065223a20317d5d5d"
    assert cmd == gobject_command

# ensure all 3 rows get created -- govobj, proposal, and event.
def test_proposal_create_and_queue(proposal):
    from models import GovernanceObject, Proposal, Event

    proposal_count = Proposal.select().count()
    event_count    = Event.select().count()
    gov_obj_count  = GovernanceObject.select().count()
    total_count = proposal_count + event_count + gov_obj_count

    assert total_count == 0

    try:
        proposal.create_and_queue()
    except PeeweeException as e:
        print "error: %s" % e[1]

    proposal_count = Proposal.select().count()
    event_count    = Event.select().count()
    gov_obj_count  = GovernanceObject.select().count()
    total_count = proposal_count + event_count + gov_obj_count

    assert proposal_count == 1
    assert event_count    == 1
    assert gov_obj_count  == 1
    assert total_count == 3

# ensure all 3 rows get created -- govobj, proposal, and event.
def test_superblock_create_and_queue(superblock):
    from models import GovernanceObject, Superblock, Event

    superblock_count = Superblock.select().count()
    event_count    = Event.select().count()
    gov_obj_count  = GovernanceObject.select().count()
    total_count = superblock_count + event_count + gov_obj_count

    assert total_count == 0

    try:
        superblock.create_and_queue()
    except PeeweeException as e:
        print "error: %s" % e[1]

    superblock_count = Superblock.select().count()
    event_count    = Event.select().count()
    gov_obj_count  = GovernanceObject.select().count()
    total_count = superblock_count + event_count + gov_obj_count

    assert superblock_count == 1
    assert event_count      == 1
    assert gov_obj_count    == 1
    assert total_count      == 3
<|endoftext|>


<commit_before>import csv
import sys
import numpy as np
#import psycopg2

#Takes in the csv file
csvFile = sys.argv[1]

def get_column(A, colNum):
    return A[:,colNum]

dataArray = [] #Array of IDs
sampArray = [] #Sample IDs
nameArray = [] #names
grainArray = []
vGeoArray = []
resArray = []
rangArray = []
formArray = []
compArray = []
wavelens = []  #matrix of num_data_point rows by 1+num_samples columns
#reflectance = []
A = np.array([])

with open(csvFile, 'rb') as cf:
    reader = csv.reader(cf)
    origin = reader.next()[1]
    print(origin)
    collection = reader.next()[1]
    desc = reader.next()[1]
    access = reader.next()[1]
    reader.next()

    data = reader.next()
    i = 2
    while i < len(data):
        #dataArray[j] = data[i]
        dataArray.append(data[i])
        i+=1

    samp = reader.next()
    i = 2
    while i < len(samp):
        sampArray.append(samp[i])
        i+=1

    name = reader.next()
    i = 2
    while i < len(name):
        nameArray.append(name[i])
        i+=1

    scale = "nanometers"
    size = reader.next()
    if ("um" or "micron") in size[0]:
        scale = "microns"
    i = 2

    while i < len(size):
        arr = []
        if ("um" or "micron") in size[i]:
            temp = size[i].split()
            temp1 = temp[0]
            if "<" in temp1:
                temp2 = temp1.replace("<", "")
                if temp2.find("um") != -1:
                    temp3 = temp2.split("u")
                    arr.append(str(float(temp3[0])*1000))
                else:
                    temp3 = float(temp2)*1000
                    arr.append(temp3)
            elif "-" in temp1:
                temp2 = temp1.split("-")
                #temp1 = str(float(temp2[0])*1000) + "-" + str(float(temp2[1])*1000)
                temp3 = str(float(temp2[0])*1000)
                arr.append(temp3)

                if temp2[1].find("um") != -1:
                    temp5 = temp2[1].split("u")
                    temp4 = str(float(temp5[0])*1000)
                else:
                    temp4 = str(float(temp2[1])*1000)
                arr.append(temp4)
            else:
                arr.append(temp1)
        else:
            #grainArray.append(size[i])
            temp = size[i].split()
            temp1 = temp[0]
            if "<" in temp1:
                temp1 = temp1.replace("<", "")
                if temp1.find("nm") != -1:
                    temp2 = temp1.split("n")
                    arr.append(temp2[0])
                else:
                    arr.append(temp1)
                #temp1 = "<" + str(temp1)
            elif "-" in temp1:
                temp2 = temp1.split("-")
                arr.append(temp2[0])
                if temp2[1].find("nm") != -1:
                    temp3 = temp2[1].split("n")
                    arr.append(temp3[0])
                else:
                    arr.append(temp2[1])
            else:
                arr.append(temp1)
        grainArray.append(arr)
        i+=1
    scale = "nanometers"

    vg = reader.next()
    i = 2
    while i < len(vg):
        if vg[i] != "":
            token = vg[i].split('/')
            vGeoArray.append(int(token[0]))
            vGeoArray.append(int(token[1]))
        else:
            #empty case - don't put in database
            vGeoArray.append(vg[i])
        i+=1

    res = reader.next()
    i = 2
    if ("um" or "micron") in res[0]:
        scale = "microns"
    else:
        scale = "nanometers"
    while i < len(res):
        if res[i] != "":
            tempArray = []
            temp1 = res[i].split('-')
            first = temp1[0].split('@')
            last = temp1[1].split('@')
            if last[1].find("nm") != -1:
                lt = last[1].split("n")
                scale = "nanometers"
            else:
                lt = last[1].split("u")
                scale = "microns"
            if scale == "microns":
                tempArray.append(str(float(first[0])*1000))
                tempArray.append(str(float(first[1])*1000))
                tempArray.append(str(float(last[0])*1000))
                tempArray.append(str(float(lt[0])*1000))
            else:
                tempArray.append(first[0])
                tempArray.append(first[1])
                tempArray.append(last[0])
                tempArray.append(lt[0])

            resArray.append(tempArray)
        else:
            #empty case- don't put in database
            resArray.append(res[i])
        i+=1

    rang = reader.next()
    i = 2
    print rang[0]
    if ("um" or "micron") in rang[0]:
        scale = "microns"
    else:
        scale = "nanometers"
    while i < len(rang):
        if rang[i] != "":
            temp1 = []
            if scale == "microns":
                temp = rang[i].split('-')
                if temp[1].find("um") != -1:
                    ty = temp[1].split("u")
                    temp1.append(str(float(temp[0])*1000))
                    temp1.append(str(float(ty[0])*1000))
                else:
                    temp1.append(str(float(temp[0])*1000))
                    temp1.append(str(float(temp[1])*1000))
            else:
                temp = rang[i].split('-')
                if temp[1].find("nm") != -1:
                    ty = temp[1].split("n")
                    temp1.append(temp[0])
                    temp1.append(ty[0])
                else:
                    temp1.append(temp[0])
                    temp1.append(temp[1])
            rangArray.append(temp1)
        else:
            #empty case - don't put in database
            rangArray.append(rang[i])
        i+=1
    scale = "nanometers"

    formula = reader.next()
    i = 2
    while i < len(formula):
        formArray.append(formula[i])
        i+=1

    comp = reader.next()
    i = 2
    while i < len(comp):
        print str(comp[i]) + "\n"
        compArray.append(comp[i])
        i+=1

    line = reader.next()
    print line
    while ("Wavelength" not in line[0]):
        print line
        line = reader.next()

    # reader.next()
    # reader.next()
    # reader.next()
    # reader.next()

    factor = 1
    wl = reader.next()
    print "wl: " + str(wl)
    if ("microns" or "um") in wl:
        factor = 1000
    for row in reader:
        #print row
        #w = reader.next()
        i = 0
        tempArray = []
        tempArray.append(str(float(row[0])*factor))
        i = 2
        while i < len(row):
            tempArray.append(str(float(row[i])*factor))
            i+=1
        wavelens.append(tempArray)
        #print tempArray
    A = np.array(wavelens)

<<<<<<< HEAD

# #----Database Things Happen Here----
#
# #Need to fill in with correct information
# conn = psycopg2.connect("dbname = 'spectrodb' user = 'dbuser' host = localhost password = ''")
# cur = conn.cursor()
#
# wave = get_column(A, 0)
# size = len(dataArray)
# for i in size:
#     dataId = dataArray[i]
#     sampId = sampArray[i]
#     name = nameArray[i]
#     gr = grainArray[i]
#     vg = vGeoArray[i]
#     res = resArray[i]
#     ran = rangArray[i]
#     form = formArray[i]
#     comp = compArray[i]
#     reflect = get_column(A, i+1)
#     query = "INSERT INTO samples (DATA_ID, SAMPLE_ID, DATE_ADDED, ORIGIN, LOCALITY, NAME, SAMPLE_DESC, GRAIN_SIZE, VIEW_GEOM, RESOLUTION, RANGE, FORMULA, COMPOSITION, WAVE_LENGTH, REFLECTANCE) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
#     data = (dataId, sampId, access, origin, collection, name, desc, gr, vg, res, ran, form, comp, wave, reflect)
#     cursor.execute(query, data)
#
# conn.commit()
# conn.close()
=======
f = open('marsdb.sql', 'w')
insert = "INSERT INTO mars_sample (data_id, sample_id, date_accessed, origin, name, grain_size, view_geom)"

for i in range(len(dataArray)):
    line = insert + " VALUES ('" + dataArray[i] + "', '" \
    + sampArray[i] + "', '" + access + "', '" + origin + "', '" \
    + nameArray[i] + "', '" + grainArray[i] + "', '{" + str(vGeoArray[i][0]) \
    + ', ' + str(vGeoArray[i][1]) + "}');"
    f.write(line + '\n')

f.close()
>>>>>>> ac863813197d0d3fe5dd4edf1d04f0a347acd3a9
<commit_msg>Didn't change anything, but wants me to commit anyway for some reason<commit_after>import csv
import sys
import numpy as np
#import psycopg2

#Takes in the csv file
csvFile = sys.argv[1]

def get_column(A, colNum):
    return A[:,colNum]

dataArray = [] #Array of IDs
sampArray = [] #Sample IDs
nameArray = [] #names
grainArray = []
vGeoArray = []
resArray = []
rangArray = []
formArray = []
compArray = []
wavelens = []  #matrix of num_data_point rows by 1+num_samples columns
#reflectance = []
A = np.array([])

with open(csvFile, 'rb') as cf:
    reader = csv.reader(cf)
    origin = reader.next()[1]
    print(origin)
    collection = reader.next()[1]
    desc = reader.next()[1]
    access = reader.next()[1]
    reader.next()

    data = reader.next()
    i = 2
    while i < len(data):
        #dataArray[j] = data[i]
        dataArray.append(data[i])
        i+=1

    samp = reader.next()
    i = 2
    while i < len(samp):
        sampArray.append(samp[i])
        i+=1

    name = reader.next()
    i = 2
    while i < len(name):
        nameArray.append(name[i])
        i+=1

    scale = "nanometers"
    size = reader.next()
    if ("um" or "micron") in size[0]:
        scale = "microns"
    i = 2

    while i < len(size):
        arr = []
        if ("um" or "micron") in size[i]:
            temp = size[i].split()
            temp1 = temp[0]
            if "<" in temp1:
                temp2 = temp1.replace("<", "")
                if temp2.find("um") != -1:
                    temp3 = temp2.split("u")
                    arr.append(str(float(temp3[0])*1000))
                else:
                    temp3 = float(temp2)*1000
                    arr.append(temp3)
            elif "-" in temp1:
                temp2 = temp1.split("-")
                #temp1 = str(float(temp2[0])*1000) + "-" + str(float(temp2[1])*1000)
                temp3 = str(float(temp2[0])*1000)
                arr.append(temp3)

                if temp2[1].find("um") != -1:
                    temp5 = temp2[1].split("u")
                    temp4 = str(float(temp5[0])*1000)
                else:
                    temp4 = str(float(temp2[1])*1000)
                arr.append(temp4)
            else:
                arr.append(temp1)
        else:
            #grainArray.append(size[i])
            temp = size[i].split()
            temp1 = temp[0]
            if "<" in temp1:
                temp1 = temp1.replace("<", "")
                if temp1.find("nm") != -1:
                    temp2 = temp1.split("n")
                    arr.append(temp2[0])
                else:
                    arr.append(temp1)
                #temp1 = "<" + str(temp1)
            elif "-" in temp1:
                temp2 = temp1.split("-")
                arr.append(temp2[0])
                if temp2[1].find("nm") != -1:
                    temp3 = temp2[1].split("n")
                    arr.append(temp3[0])
                else:
                    arr.append(temp2[1])
            else:
                arr.append(temp1)
        grainArray.append(arr)
        i+=1
    scale = "nanometers"

    vg = reader.next()
    i = 2
    while i < len(vg):
        if vg[i] != "":
            token = vg[i].split('/')
            vGeoArray.append(int(token[0]))
            vGeoArray.append(int(token[1]))
        else:
            #empty case - don't put in database
            vGeoArray.append(vg[i])
        i+=1

    res = reader.next()
    i = 2
    if ("um" or "micron") in res[0]:
        scale = "microns"
    else:
        scale = "nanometers"
    while i < len(res):
        if res[i] != "":
            tempArray = []
            temp1 = res[i].split('-')
            first = temp1[0].split('@')
            last = temp1[1].split('@')
            if last[1].find("nm") != -1:
                lt = last[1].split("n")
                scale = "nanometers"
            else:
                lt = last[1].split("u")
                scale = "microns"
            if scale == "microns":
                tempArray.append(str(float(first[0])*1000))
                tempArray.append(str(float(first[1])*1000))
                tempArray.append(str(float(last[0])*1000))
                tempArray.append(str(float(lt[0])*1000))
            else:
                tempArray.append(first[0])
                tempArray.append(first[1])
                tempArray.append(last[0])
                tempArray.append(lt[0])

            resArray.append(tempArray)
        else:
            #empty case- don't put in database
            resArray.append(res[i])
        i+=1

    rang = reader.next()
    i = 2
    print rang[0]
    if ("um" or "micron") in rang[0]:
        scale = "microns"
    else:
        scale = "nanometers"
    while i < len(rang):
        if rang[i] != "":
            temp1 = []
            if scale == "microns":
                temp = rang[i].split('-')
                if temp[1].find("um") != -1:
                    ty = temp[1].split("u")
                    temp1.append(str(float(temp[0])*1000))
                    temp1.append(str(float(ty[0])*1000))
                else:
                    temp1.append(str(float(temp[0])*1000))
                    temp1.append(str(float(temp[1])*1000))
            else:
                temp = rang[i].split('-')
                if temp[1].find("nm") != -1:
                    ty = temp[1].split("n")
                    temp1.append(temp[0])
                    temp1.append(ty[0])
                else:
                    temp1.append(temp[0])
                    temp1.append(temp[1])
            rangArray.append(temp1)
        else:
            #empty case - don't put in database
            rangArray.append(rang[i])
        i+=1
    scale = "nanometers"

    formula = reader.next()
    i = 2
    while i < len(formula):
        formArray.append(formula[i])
        i+=1

    comp = reader.next()
    i = 2
    while i < len(comp):
        print str(comp[i]) + "\n"
        compArray.append(comp[i])
        i+=1

    line = reader.next()
    print line
    while ("Wavelength" not in line[0]):
        print line
        line = reader.next()

    if ("microns" or "um") in line:
        factor = 1000
    # reader.next()
    # reader.next()
    # reader.next()
    # reader.next()

    factor = 1
    wl = reader.next()
    print "wl: " + str(wl)

    for row in reader:
        #print row
        #w = reader.next()
        i = 0
        tempArray = []
        tempArray.append(str(float(row[0])*factor))
        i = 2
        while i < len(row):
            tempArray.append(str(float(row[i])*factor))
            i+=1
        wavelens.append(tempArray)
        #print tempArray
    A = np.array(wavelens)

<<<<<<< HEAD

# #----Database Things Happen Here----
#
# #Need to fill in with correct information
# conn = psycopg2.connect("dbname = 'spectrodb' user = 'dbuser' host = localhost password = ''")
# cur = conn.cursor()
#
# wave = get_column(A, 0)
# size = len(dataArray)
# for i in size:
#     dataId = dataArray[i]
#     sampId = sampArray[i]
#     name = nameArray[i]
#     gr = grainArray[i]
#     vg = vGeoArray[i]
#     res = resArray[i]
#     ran = rangArray[i]
#     form = formArray[i]
#     comp = compArray[i]
#     reflect = get_column(A, i+1)
#     query = "INSERT INTO samples (DATA_ID, SAMPLE_ID, DATE_ADDED, ORIGIN, LOCALITY, NAME, SAMPLE_DESC, GRAIN_SIZE, VIEW_GEOM, RESOLUTION, RANGE, FORMULA, COMPOSITION, WAVE_LENGTH, REFLECTANCE) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
#     data = (dataId, sampId, access, origin, collection, name, desc, gr, vg, res, ran, form, comp, wave, reflect)
#     cursor.execute(query, data)
#
# conn.commit()
# conn.close()
=======
f = open('marsdb.sql', 'w')
insert = "INSERT INTO mars_sample (data_id, sample_id, date_accessed, origin, name, grain_size, view_geom)"

for i in range(len(dataArray)):
    line = insert + " VALUES ('" + dataArray[i] + "', '" \
    + sampArray[i] + "', '" + access + "', '" + origin + "', '" \
    + nameArray[i] + "', '" + grainArray[i] + "', '{" + str(vGeoArray[i][0]) \
    + ', ' + str(vGeoArray[i][1]) + "}');"
    f.write(line + '\n')

f.close()
>>>>>>> ac863813197d0d3fe5dd4edf1d04f0a347acd3a9
<|endoftext|>


<commit_before>import pbclient
import json
from helpers import *
from default import TestDefault
from mock import patch, MagicMock
from nose.tools import assert_raises
from requests import exceptions
from pbsexceptions import ProjectNotFound

class TestPbsUpdateProject(TestDefault):

    """Test class for pbs update project commands."""

    @patch('helpers.find_project_by_short_name')
    def test_update_project_create(self, find_mock):
        """Test update_project works."""
        project = MagicMock()
        project.name = 'name'
        project.short_name = 'short_name'
        project.description = 'description'
        project.info = dict()

        find_mock.return_value = project

        task_presenter = MagicMock()
        task_presenter.read.return_value = "presenter"

        tutorial = MagicMock()
        tutorial.read.return_value = "tutorial"

        long_description = MagicMock()
        long_description.read.return_value = "long_description"

        pbclient = MagicMock()
        pbclient.update_project.return_value = {'short_name': 'short_name'}
        self.config.pbclient = pbclient
        res = _update_project(self.config, task_presenter,
                              long_description, tutorial)
        assert res == 'Project short_name updated!', res

    @patch('helpers.find_project_by_short_name')
    def test_update_project_connection_error(self, find_mock):
        """Test update_project connection error works."""
        project = MagicMock()
        project.name = 'name'
        project.short_name = 'short_name'
        project.description = 'description'
        project.info = dict()

        find_mock.return_value = project

        task_presenter = MagicMock()
        task_presenter.read.return_value = "presenter"

        tutorial = MagicMock()
        tutorial.read.return_value = "tutorial"

        long_description = MagicMock()
        long_description.read.return_value = "long_description"

        pbclient = MagicMock()
        pbclient.update_project.side_effect = exceptions.ConnectionError
        self.config.pbclient = pbclient
        res = _update_project(self.config, task_presenter,
                              long_description, tutorial)
        assert res == "Connection Error! The server http://server is not responding", res

    @patch('helpers.find_project_by_short_name')
    def test_update_project_another_error(self, find_mock):
        """Test update_project another error works."""
        project = MagicMock()
        project.name = 'name'
        project.short_name = 'short_name'
        project.description = 'description'
        project.info = dict()

        find_mock.return_value = project

        task_presenter = MagicMock()
        task_presenter.read.return_value = "presenter"

        tutorial = MagicMock()
        tutorial.read.return_value = "tutorial"

        long_description = MagicMock()
        long_description.read.return_value = "long_description"

        pbclient = MagicMock()
        print self.error
        pbclient.update_project.return_value = self.error
        self.config.pbclient = pbclient
        assert_raises(ProjectNotFound, _update_project, self.config,
                      task_presenter, long_description, tutorial)
<commit_msg>Fixes.<commit_after>import pbclient
import json
from helpers import *
from default import TestDefault
from mock import patch, MagicMock
from nose.tools import assert_raises
from requests import exceptions
from pbsexceptions import ProjectNotFound

class TestPbsUpdateProject(TestDefault):

    """Test class for pbs update project commands."""

    @patch('helpers.find_project_by_short_name')
    def test_update_project_create(self, find_mock):
        """Test update_project works."""
        project = MagicMock()
        project.name = 'name'
        project.short_name = 'short_name'
        project.description = 'description'
        project.info = dict()

        find_mock.return_value = project

        task_presenter = "test/template.html"

        tutorial = "test/tutorial.html"

        long_description = "test/long_description.md"

        pbclient = MagicMock()
        pbclient.update_project.return_value = {'short_name': 'short_name'}
        self.config.pbclient = pbclient
        res = _update_project(self.config, task_presenter,
                              long_description, tutorial)
        assert res == 'Project short_name updated!', res

    @patch('helpers.find_project_by_short_name')
    def test_update_project_connection_error(self, find_mock):
        """Test update_project connection error works."""
        project = MagicMock()
        project.name = 'name'
        project.short_name = 'short_name'
        project.description = 'description'
        project.info = dict()

        find_mock.return_value = project

        task_presenter = "test/template.html"

        tutorial = "test/tutorial.html"

        long_description = "test/long_description.md"

        pbclient = MagicMock()
        pbclient.update_project.side_effect = exceptions.ConnectionError
        self.config.pbclient = pbclient
        res = _update_project(self.config, task_presenter,
                              long_description, tutorial)
        assert res == "Connection Error! The server http://server is not responding", res

    @patch('helpers.find_project_by_short_name')
    def test_update_project_another_error(self, find_mock):
        """Test update_project another error works."""
        project = MagicMock()
        project.name = 'name'
        project.short_name = 'short_name'
        project.description = 'description'
        project.info = dict()

        find_mock.return_value = project

        task_presenter = "test/template.html"

        tutorial = "test/tutorial.html"

        long_description = "test/long_description.md"

        pbclient = MagicMock()
        print self.error
        pbclient.update_project.return_value = self.error
        self.config.pbclient = pbclient
        assert_raises(ProjectNotFound, _update_project, self.config,
                      task_presenter, long_description, tutorial)
<|endoftext|>