<commit_before>#! /usr/bin/env python
# -*- encoding: UTF-8 -*-

# DOCUMENTATION
#http://doc.aldebaran.com/2-5/naoqi/peopleperception/alpeopleperception-api.html#PeoplePerception
#http://doc.aldebaran.com/2-5/naoqi/peopleperception/alfacedetection.html

'''
Memorize people
==================

Writes in ALMemory 'Actions/MemorizePeople/PeopleList/' json string with all features

params== 'SPRgame'
'''

import qi
import argparse
import sys
import os
import time
import threading
import json
import math

from naoqi import ALProxy

import action_base
from action_base import *

from utils import point2world
import conditions
from conditions import set_condition

import headpose

actionName = "memorizepeople"

global people_list
global memory_service

Step_turn_angle= math.pi/5
Max_turn_angle= math.pi
global countdt
countdt = 0
global currentangle
currentangle=0
Timeoutangle=10

headYaw = [ 0.9, 0.5, 0.0, -0.5, -0.5, 0.0, 0.5,0.9, 0.0 ]
headPitch = [ -0.3, -0.3, -0.3, -0.3, 0.0, 0.0, 0.0 ,0.0, -0.1 ]
headtime = 0.8

def update_data(currentuser):

    b_new=True

    try:
        mem_list=memory_service.getData('Actions/MemorizePeople/PeopleList')
        people_list=json.loads(mem_list)

        if len(people_list)>1:
            # update some info for old detections

            for i in range(len(people_list)):

                if currentuser['person_naoqiid']==people_list[i]['person_naoqiid']:

                    json_person=people_list[i]

        #            mem_person=memory_service.getData('Actions/MemorizePeople/'+people_list[i])
        #            json_person=json.loads(mem_person)
                    b_new=False

                    json_person['info']['height']=currentuser['info']['height']
                    json_person['info']['posture']=currentuser['info']['posture']

                    # Update only if confidence is higher
                    if currentuser['face_naoqi']['faceinfo']['age']['conf'] >json_person['face_naoqi']['faceinfo']['age']['conf']:
                        json_person['face_naoqi']['faceinfo']['age']=currentuser['face_naoqi']['faceinfo']['age']

                    if currentuser['face_naoqi']['faceinfo']['gender']['conf'] >json_person['face_naoqi']['faceinfo']['gender']['conf']:
                        json_person['face_naoqi']['faceinfo']['gender']=currentuser['face_naoqi']['faceinfo']['gender']

        #            ## Write data in ALMemory
        #            str_person=json.dumps(json_person)
        #            memory_service.insertData('Actions/MemorizePeople/Person/'+currentuser['personid'], str_person)

                    people_list[i]=json_person

    except:
        pass

    ## ADD new person
    if b_new is True:

        people_list=[]

        people_list.append(currentuser)

    str_person=json.dumps(people_list)

    memory_service.insertData('Actions/MemorizePeople/Peoplelist', str_person)

def actionThread_exec (params):

    #global face_char_service
    global memory_service
    t = threading.currentThread()
    memory_service = getattr(t, "mem_serv", None)
    faces_service = getattr(t, "session", None).service("ALFaceDetection")
    faces_service.setRecognitionEnabled(True)
    motion_service  = getattr(t, "session", None).service("ALMotion")
    face_char_service = getattr(t, "session", None).service("ALFaceCharacteristics")
    print "Action "+actionName+" started with params "+params

    # DElete old list
    global people_list
    people_list=[]

    b_completed=False
    memory_service.insertData('Actions/Memorizepeople/Peoplelist',people_list)

    while (getattr(t, "do_run", True) and b_completed==False):

        motion_service
        naoqi_people_list =  memory_service.getData("PeoplePerception/PeopleList")

        person=None
        poseinworld=None
#        print 'naoqi_people_list= ',naoqi_people_list
#        print '#################'
        for personid in naoqi_people_list:

            #print 'personid= ',personid
#            print '__________'
            face={'name':'','faceinfo':{}}
            age={'val':0.0,'conf':0.0}
            gender={'val':0.0,'conf':0.0}
            smile={'val':0.0,'conf':0.0}
            try:

                #try:
                res_char=face_char_service.analyzeFaceCharacteristics(personid)
                print 'res_char',res_char

                #lookingat =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/LookingAtRobotScore")
                #gazedirection =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/GazeDirection")
                propage =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/AgeProperties")
                propgender =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/GenderProperties")
                propexpression =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/ExpressionProperties")
                propsmile =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/SmileProperties")

                print 'propgender',propgender

                if propgender[0] ==0.0:
                    str_gender='female'
                elif propgender[0] ==1.0:
                    str_gender='male'
                age={'val':propage[0],'conf':round(propage[1],2)}
                gender={'val':str_gender,'conf':round(propgender[1],2)}
                smile={'val':round(propsmile[0],2),'conf':round(propsmile[1],2)}

                #propexpression  [neutral, happy, surprised, angry or sad].
                #expression={'neutral':round(propexpression[0],2),'happy':round(propexpression[1],2),'surprised':round(propexpression[2],2),'angry':round(propexpression[3],2),'sad':round(propexpression[4],2)}

            except:
                print 'FaceCharacteristics error '
                pass
#
            facecharacteristics={'age':age, 'gender':gender,'smile':smile, 'expression':{}}

            shirtcolor={'name': '', 'hsv':[]}
            personinfo={'height': 0.0, 'shirtcolor': shirtcolor,'posture':'' }
            poseinworld={'x':0.0,'y': 0.0}

            try:
#                angles =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/AnglesYawPitch")
#                distance =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/Distance")
                height =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/RealHeight")
                shirtcolorName =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/ShirtColor")
                shirtcolorHSV =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/ShirtColorHSV")
                PositionInRobotFrame =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/PositionInRobotFrame")
                issitting=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsSitting") #0 is standing,1 sitting,2 is unknown.

                iswaving=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsWaving")
                iswavingcenter=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsWavingCenter")
                iswavingleft=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsWavingLeft")
                iswavingright=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsWavingRight")

#
                posture='unknow'
                if issitting==0:
                    posture='standing'
                elif issitting==1:
                    posture='sitting'

                wavingmode='none'
                if iswavingcenter==1:
                    wavingmode='center'
                if iswavingcenter==1:
                    iswavingleft='left'
                if iswavingcenter==1:
                    iswavingright='right'
                # Write data in json format

                shirtcolor={'name': shirtcolorName, 'hsv':shirtcolorHSV}
                personinfo={'height': round(height,2), 'shirtcolor': shirtcolor,'posture':posture,'waving': wavingmode}

                w_px, w_py = point2world(memory_service,[PositionInRobotFrame[0],PositionInRobotFrame[1]])
                poseinworld={'x':w_px,'y': w_py}

            except:
                print 'Person info error '
                pass

            try:
                current_node=memory_service.getData('TopologicalNav/CurrentNode')
                closest_node=memory_service.getData('TopologicalNav/ClosestNode')

            except:
                #print 'topological localization error '
                current_node=None
                closest_node=None

            face['faceinfo']=facecharacteristics

            posetopological={'current_node':current_node,'closest_node':closest_node}
            lastlocation= {'world':poseinworld , 'topological':posetopological}

            user={'personid': personid ,'person_naoqiid': personid,'info': personinfo, 'lastlocation':lastlocation, 'face_naoqi': face,'face_ms_api': {}}

            update_data(user)

        if params== 'SPRgame':
            global countdt
            global currentangle

            if countdt>Timeoutangle:

#                motion_service.moveTo(0.0, 0.0, currentangle)
#                currentangle+=Step_turn_angle

                headpose.moveHead(motion_service, headYaw[currentangle], headPitch[currentangle], headtime)
                countdt = 0
                currentangle+=1
            else:
                countdt+=1

            if currentangle > len(headYaw)-1:
                print 'stop currentangle=',currentangle

                b_completed=True

#            if currentangle > Max_turn_angle:
#                print 'stop currentangle=',currentangle
#
#                b_completed=True

#            try:
#                mem_list=memory_service.getData('Actions/MemorizePeople/PeopleList')
#                people_list=json.loads(mem_list)
#
#                b_completed=True
#
#                for p in people_list:
#                    if p['face_naoqi']['faceinfo']['gender']['conf']<0.6:
#                        b_completed=False
#
#                if b_completed is True and len(people_list)>3:
#                    b_completed=True
#
#
#            except:
#                pass
#

        time.sleep(0.3)

    memory_service.raiseEvent("PNP_action_result_"+actionName,"success");
    print "Action "+actionName+" "+params+" terminated"

def init(session):
    print actionName+" init"
    action_base.init(session, actionName, actionThread_exec)

def quit():
    print actionName+" quit"
    actionThread_exec.do_run = False

if __name__ == "__main__":

    app = action_base.initApp(actionName)

    init(app.session)

    #Program stays at this point until we stop it
    app.run()

    quit()
<commit_msg>Update memorizepeople.py<commit_after>#! /usr/bin/env python
# -*- encoding: UTF-8 -*-

# DOCUMENTATION
#http://doc.aldebaran.com/2-5/naoqi/peopleperception/alpeopleperception-api.html#PeoplePerception
#http://doc.aldebaran.com/2-5/naoqi/peopleperception/alfacedetection.html

'''
Memorize people
==================

Writes in ALMemory 'Actions/MemorizePeople/PeopleList/' json string with all features

params== 'SPRgame'
'''

import qi
import argparse
import sys
import os
import time
import threading
import json
import math

from naoqi import ALProxy

import action_base
from action_base import *

from utils import point2world
import conditions
from conditions import set_condition

import headpose

actionName = "memorizepeople"

global people_list
global memory_service

Step_turn_angle= math.pi/5
Max_turn_angle= math.pi
global countdt
countdt = 0
global currentangle
currentangle=0
Timeoutangle=10

headYaw = [ 0.9, 0.5, 0.0, -0.5, -0.5, 0.0, 0.5,0.9, 0.0 ]
headPitch = [ -0.3, -0.3, -0.3, -0.3, 0.0, 0.0, 0.0 ,0.0, -0.1 ]
headtime = 0.8

def update_data(currentuser):

    b_new=True

    try:
        mem_list=memory_service.getData('Actions/MemorizePeople/PeopleList')
        people_list=json.loads(mem_list)

        if len(people_list)>1:
            # update some info for old detections

            for i in range(len(people_list)):

                if currentuser['person_naoqiid']==people_list[i]['person_naoqiid']:

                    json_person=people_list[i]

        #            mem_person=memory_service.getData('Actions/MemorizePeople/'+people_list[i])
        #            json_person=json.loads(mem_person)
                    b_new=False

                    json_person['info']['height']=currentuser['info']['height']
                    json_person['info']['posture']=currentuser['info']['posture']

                    # Update only if confidence is higher
                    if currentuser['face_naoqi']['faceinfo']['age']['conf'] >json_person['face_naoqi']['faceinfo']['age']['conf']:
                        json_person['face_naoqi']['faceinfo']['age']=currentuser['face_naoqi']['faceinfo']['age']

                    if currentuser['face_naoqi']['faceinfo']['gender']['conf'] >json_person['face_naoqi']['faceinfo']['gender']['conf']:
                        json_person['face_naoqi']['faceinfo']['gender']=currentuser['face_naoqi']['faceinfo']['gender']

        #            ## Write data in ALMemory
        #            str_person=json.dumps(json_person)
        #            memory_service.insertData('Actions/MemorizePeople/Person/'+currentuser['personid'], str_person)

                    people_list[i]=json_person

    except:
        people_list=[]
        pass

    ## ADD new person
    if b_new is True:

        people_list.append(currentuser)

    str_person=json.dumps(people_list)

    memory_service.insertData('Actions/MemorizePeople/Peoplelist', str_person)

def actionThread_exec (params):

    #global face_char_service
    global memory_service
    t = threading.currentThread()
    memory_service = getattr(t, "mem_serv", None)
    faces_service = getattr(t, "session", None).service("ALFaceDetection")
    faces_service.setRecognitionEnabled(True)
    motion_service  = getattr(t, "session", None).service("ALMotion")
    face_char_service = getattr(t, "session", None).service("ALFaceCharacteristics")
    print "Action "+actionName+" started with params "+params

    # DElete old list
    global people_list
    people_list=[]

    b_completed=False
    memory_service.insertData('Actions/Memorizepeople/Peoplelist',people_list)

    while (getattr(t, "do_run", True) and b_completed==False):

        motion_service
        naoqi_people_list =  memory_service.getData("PeoplePerception/PeopleList")

        person=None
        poseinworld=None
#        print 'naoqi_people_list= ',naoqi_people_list
#        print '#################'
        for personid in naoqi_people_list:

            #print 'personid= ',personid
#            print '__________'
            face={'name':'','faceinfo':{}}
            age={'val':0.0,'conf':0.0}
            gender={'val':0.0,'conf':0.0}
            smile={'val':0.0,'conf':0.0}
            try:

                #try:
                res_char=face_char_service.analyzeFaceCharacteristics(personid)
                print 'res_char',res_char

                #lookingat =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/LookingAtRobotScore")
                #gazedirection =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/GazeDirection")
                propage =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/AgeProperties")
                propgender =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/GenderProperties")
                propexpression =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/ExpressionProperties")
                propsmile =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/SmileProperties")

                print 'propgender',propgender

                if propgender[0] ==0.0:
                    str_gender='female'
                elif propgender[0] ==1.0:
                    str_gender='male'
                age={'val':propage[0],'conf':round(propage[1],2)}
                gender={'val':str_gender,'conf':round(propgender[1],2)}
                smile={'val':round(propsmile[0],2),'conf':round(propsmile[1],2)}

                #propexpression  [neutral, happy, surprised, angry or sad].
                #expression={'neutral':round(propexpression[0],2),'happy':round(propexpression[1],2),'surprised':round(propexpression[2],2),'angry':round(propexpression[3],2),'sad':round(propexpression[4],2)}

            except:
                print 'FaceCharacteristics error '
                pass
#
            facecharacteristics={'age':age, 'gender':gender,'smile':smile, 'expression':{}}

            shirtcolor={'name': '', 'hsv':[]}
            personinfo={'height': 0.0, 'shirtcolor': shirtcolor,'posture':'' }
            poseinworld={'x':0.0,'y': 0.0}

            try:
#                angles =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/AnglesYawPitch")
#                distance =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/Distance")
                height =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/RealHeight")
                shirtcolorName =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/ShirtColor")
                shirtcolorHSV =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/ShirtColorHSV")
                PositionInRobotFrame =memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/PositionInRobotFrame")
                issitting=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsSitting") #0 is standing,1 sitting,2 is unknown.

                iswaving=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsWaving")
                iswavingcenter=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsWavingCenter")
                iswavingleft=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsWavingLeft")
                iswavingright=memory_service.getData( "PeoplePerception/Person/" +str(personid)+"/IsWavingRight")

#
                posture='unknow'
                if issitting==0:
                    posture='standing'
                elif issitting==1:
                    posture='sitting'

                wavingmode='none'
                if iswavingcenter==1:
                    wavingmode='center'
                if iswavingcenter==1:
                    iswavingleft='left'
                if iswavingcenter==1:
                    iswavingright='right'
                # Write data in json format

                shirtcolor={'name': shirtcolorName, 'hsv':shirtcolorHSV}
                personinfo={'height': round(height,2), 'shirtcolor': shirtcolor,'posture':posture,'waving': wavingmode}

                w_px, w_py = point2world(memory_service,[PositionInRobotFrame[0],PositionInRobotFrame[1]])
                poseinworld={'x':w_px,'y': w_py}

            except:
                print 'Person info error '
                pass

            try:
                current_node=memory_service.getData('TopologicalNav/CurrentNode')
                closest_node=memory_service.getData('TopologicalNav/ClosestNode')

            except:
                #print 'topological localization error '
                current_node=None
                closest_node=None

            face['faceinfo']=facecharacteristics

            posetopological={'current_node':current_node,'closest_node':closest_node}
            lastlocation= {'world':poseinworld , 'topological':posetopological}

            user={'personid': personid ,'person_naoqiid': personid,'info': personinfo, 'lastlocation':lastlocation, 'face_naoqi': face,'face_ms_api': {}}

            update_data(user)

        if params== 'SPRgame':
            global countdt
            global currentangle

            if countdt>Timeoutangle:

#                motion_service.moveTo(0.0, 0.0, currentangle)
#                currentangle+=Step_turn_angle

                headpose.moveHead(motion_service, headYaw[currentangle], headPitch[currentangle], headtime)
                countdt = 0
                currentangle+=1
            else:
                countdt+=1

            if currentangle > len(headYaw)-1:
                print 'stop currentangle=',currentangle

                b_completed=True

#            if currentangle > Max_turn_angle:
#                print 'stop currentangle=',currentangle
#
#                b_completed=True

#            try:
#                mem_list=memory_service.getData('Actions/MemorizePeople/PeopleList')
#                people_list=json.loads(mem_list)
#
#                b_completed=True
#
#                for p in people_list:
#                    if p['face_naoqi']['faceinfo']['gender']['conf']<0.6:
#                        b_completed=False
#
#                if b_completed is True and len(people_list)>3:
#                    b_completed=True
#
#
#            except:
#                pass
#

        time.sleep(0.3)

    memory_service.raiseEvent("PNP_action_result_"+actionName,"success");
    print "Action "+actionName+" "+params+" terminated"

def init(session):
    print actionName+" init"
    action_base.init(session, actionName, actionThread_exec)

def quit():
    print actionName+" quit"
    actionThread_exec.do_run = False

if __name__ == "__main__":

    app = action_base.initApp(actionName)

    init(app.session)

    #Program stays at this point until we stop it
    app.run()

    quit()
<|endoftext|>


<commit_before>__version__ = '0.9.0'
<commit_msg>update version<commit_after>__version__ = '0.10.1'
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import logging
from sendmail import sendmail
import datetime
import mailbox
import email
import report
from bot import Bot
from config import config
from db import db

logger = logging.getLogger(__name__)

class Mailbot(Bot):

    # returns a list of Report objects
    def crawl(self, user):
        reports = []
        # todo: adjust to actual mailbox
        mails = mailbox.mbox("/var/mail/" + config['mail']['mbox_user'])
        for msg in mails:
            if get_date_from_header(msg['Date']) > user.get_seen_mail():
                if user.get_city().lower() in msg['To'].lower():
                    reports.append(make_report(msg, user))
        return reports

    # post/boost Report object
    def post(self, user, report):
        recipients = user.get_mailinglist()
        for rec in recipients:
            rec = rec[0]
            unsubscribe_text = "\n_______\nYou don't want to receive those messages? Unsubscribe with this link: "
            body = report.text + unsubscribe_text + config['web']['host'] + "/city/mail/unsubscribe/" \
                   + db.mail_subscription_token(rec, user.get_city())
            if report.author != rec:
                try:
                    city = user.get_city()
                    sendmail(rec, "Ticketfrei " + city + " Report",
                                      city=city, body=body)
                except Exception:
                    logger.error("Sending Mail failed.", exc_info=True)

def make_report(msg, user):
    """
    generates a report out of a mail

    :param msg: email.parser.Message object
    :return: post: report.Report object
    """
    # get a comparable date out of the email
    date = get_date_from_header(msg['Date'])

    author = msg['From']  # get mail author from email header
    # :todo take only the part in between the < >

    if msg.is_multipart():
        text = []
        for part in msg.get_payload():
            if part.get_content_type() == "text":
                text.append(part.get_payload())
            elif part.get_content_type() == "multipart/mixed":
                for p in part:
                    if p.get_content_type() == "text":
                        text.append(part.get_payload())
                    else:
                        logger.error("unknown MIMEtype: " +
                                     p.get_content_type())
            else:
                logger.error("unknown MIMEtype: " +
                             part.get_content_type())
        text = '\n'.join(text)
    else:
        text = msg.get_payload()
    post = report.Report(author, "mail", text, None, date)
    user.save_seen_mail(date)
    return post

def get_date_from_header(header):
    """
    :param header: msg['Date']
    :return: float: total seconds
    """
    date_tuple = email.utils.parsedate_tz(header)
    date_tuple = datetime.datetime.fromtimestamp(
        email.utils.mktime_tz(date_tuple)
    )
    return (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()
<commit_msg>fix shutdown in #40<commit_after>#!/usr/bin/env python3

import logging
from sendmail import sendmail
import datetime
import mailbox
import email
import report
from bot import Bot
from config import config
from db import db

logger = logging.getLogger(__name__)

class Mailbot(Bot):

    # returns a list of Report objects
    def crawl(self, user):
        reports = []
        # todo: adjust to actual mailbox
        mails = mailbox.mbox("/var/mail/" + config['mail']['mbox_user'])
        for msg in mails:
            if get_date_from_header(msg['Date']) > user.get_seen_mail():
                if user.get_city().lower() in msg['To'].lower():
                    reports.append(make_report(msg, user))
        return reports

    # post/boost Report object
    def post(self, user, report):
        recipients = user.get_mailinglist()
        for rec in recipients:
            rec = rec[0]
            unsubscribe_text = "\n_______\nYou don't want to receive those messages? Unsubscribe with this link: "
            body = report.text + unsubscribe_text + config['web']['host'] + "/city/mail/unsubscribe/" \
                   + db.mail_subscription_token(rec, user.get_city())
            if report.author != rec:
                try:
                    city = user.get_city()
                    sendmail(rec, "Ticketfrei " + city + " Report",
                                      city=city, body=body)
                except Exception:
                    logger.error("Sending Mail failed.", exc_info=True)

def make_report(msg, user):
    """
    generates a report out of a mail

    :param msg: email.parser.Message object
    :return: post: report.Report object
    """
    # get a comparable date out of the email
    date = get_date_from_header(msg['Date'])

    author = msg['From']  # get mail author from email header
    # :todo take only the part in between the < >

    if msg.is_multipart():
        text = []
        for part in msg.get_payload():
            if part.get_content_type() == "text":
                text.append(part.get_payload())
            elif part.get_content_type() == "multipart/mixed":
                for p in part:
                    if isinstance(p, str):
                        text.append(p)
                    if p.get_content_type() == "text":
                        text.append(part.get_payload())
                    else:
                        logger.error("unknown MIMEtype: " +
                                     p.get_content_type())
            else:
                logger.error("unknown MIMEtype: " +
                             part.get_content_type())
        text = '\n'.join(text)
    else:
        text = msg.get_payload()
    post = report.Report(author, "mail", text, None, date)
    user.save_seen_mail(date)
    return post

def get_date_from_header(header):
    """
    :param header: msg['Date']
    :return: float: total seconds
    """
    date_tuple = email.utils.parsedate_tz(header)
    date_tuple = datetime.datetime.fromtimestamp(
        email.utils.mktime_tz(date_tuple)
    )
    return (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()
<|endoftext|>


<commit_before>  import sublime, sublime_plugin
import os
import subprocess
import threading

class OpenTerminalHere(sublime_plugin.TextCommand):
    def run(self, edit):
        view = self.view
        file_path = view.file_name()
        dirname = os.path.dirname(file_path)

        th = GnomeTerminalThread(dirname)
        th.start()

    def enabled(self):
        return True if self.view.file_name() else False

class GnomeTerminalThread(threading.Thread):
    def __init__(self, dirname):
        self.dirname = dirname
        threading.Thread.__init__(self)

    def run(self):
        if self.dirname:
            fpc = "--working-directory={0}".format(self.dirname)
            subprocess.call(['gnome-terminal', fpc])
<commit_msg>Update open_terminal_here.py<commit_after>import sublime, sublime_plugin
import os
import subprocess
import threading

class OpenTerminalHere(sublime_plugin.TextCommand):
    def run(self, edit):
        view = self.view
        file_path = view.file_name()
        dirname = os.path.dirname(file_path)

        th = GnomeTerminalThread(dirname)
        th.start()

    def enabled(self):
        return True if self.view.file_name() else False

class GnomeTerminalThread(threading.Thread):
    def __init__(self, dirname):
        self.dirname = dirname
        threading.Thread.__init__(self)

    def run(self):
        if self.dirname:
            fpc = "--working-directory={0}".format(self.dirname)
            subprocess.call(['gnome-terminal', fpc])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Yahoo! Finance Fix for Pandas Datareader
# https://github.com/ranaroussi/fix-yahoo-finance
#
# Copyright 2017 Ran Aroussi
#
# Licensed under the GNU Lesser General Public License, v3.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.gnu.org/licenses/lgpl-3.0.en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.0.8"
__author__ = "Ran Aroussi"
__all__ = ['download', 'get_yahoo_crumb', 'parse_ticker_csv']

import datetime
import numpy as np
import pandas as pd
import time
import io
import requests
import re
import warnings
import sys

_YAHOO_COOKIE_ = ''
_YAHOO_CRUMB_ = ''
_YAHOO_CHECKED_ = None
_YAHOO_TTL_ = 180

def get_yahoo_crumb(force=False):
    global _YAHOO_COOKIE_, _YAHOO_CRUMB_, _YAHOO_CHECKED_, _YAHOO_TTL_

    # use same cookie for 5 min
    if _YAHOO_CHECKED_ and not force:
        now = datetime.datetime.now()
        delta = (now - _YAHOO_CHECKED_).total_seconds()
        if delta < _YAHOO_TTL_:
            return (_YAHOO_CRUMB_, _YAHOO_COOKIE_)

    res = requests.get('https://finance.yahoo.com/quote/SPY/history')
    _YAHOO_COOKIE_ = res.cookies['B']

    pattern = re.compile('.*"CrumbStore":\{"crumb":"(?P<crumb>[^"]+)"\}')
    for line in res.text.splitlines():
        m = pattern.match(line)
        if m is not None:
            _YAHOO_CRUMB_ = m.groupdict()['crumb']

    # set global params
    _YAHOO_CHECKED_ = datetime.datetime.now()

    return (_YAHOO_CRUMB_, _YAHOO_COOKIE_)

def parse_ticker_csv(csv_str, auto_adjust):
    df = pd.read_csv(csv_str, index_col=0, error_bad_lines=False, warn_bad_lines=False
                     ).replace('null', np.nan).dropna()

    df.index = pd.to_datetime(df.index)
    df = df.apply(pd.to_numeric)
    df['Volume'] = df['Volume'].fillna(0).astype(int)

    if auto_adjust:
        ratio = df["Close"] / df["Adj Close"]
        df["Adj Open"] = df["Open"] / ratio
        df["Adj High"] = df["High"] / ratio
        df["Adj Low"] = df["Low"] / ratio

        df.drop(
            ["Open", "High", "Low", "Close"],
            axis=1, inplace=True)

        df.rename(columns={
            "Adj Open": "Open", "Adj High": "High",
            "Adj Low": "Low", "Adj Close": "Close"
        }, inplace=True)

        df = df[['Open', 'High', 'Low', 'Close', 'Volume']]

    return df

def download(tickers, start=None, end=None, as_panel=True,
             group_by='column', auto_adjust=False, progress=True,
             *args, **kwargs):

    # format start
    if start is None:
        start = int(time.mktime(time.strptime('1950-01-01', '%Y-%m-%d')))
    elif isinstance(start, datetime.datetime):
        start = int(time.mktime(start.timetuple()))
    else:
        start = int(time.mktime(time.strptime(str(start), '%Y-%m-%d')))

    # format end
    if end is None:
        end = int(time.mktime(datetime.datetime.now().timetuple()))
    elif isinstance(end, datetime.datetime):
        end = int(time.mktime(end.timetuple()))
    else:
        end = int(time.mktime(time.strptime(str(end), '%Y-%m-%d')))

    # iterval
    interval = kwargs["interval"] if "interval" in kwargs else "1d"

    # url template
    url_str = "https://query1.finance.yahoo.com/v7/finance/download/%s"
    url_str += "?period1=%s&period2=%s&interval=%s&events=history&crumb=%s"

    # dataframe collector
    dfs = {}

    # create ticker list
    tickers = tickers if isinstance(tickers, list) else [tickers]
    tickers = [x.upper() for x in tickers]

    # initiate progress bar
    if progress:
        pbar = ProgressBar(len(tickers), 'downloaded')

    # failed tickers collectors
    round1_failed_tickers = []
    round2_failed_tickers = []

    # start downloading
    for ticker in tickers:

        # yahoo crumb/cookie
        crumb, cookie = get_yahoo_crumb()

        tried_once = False
        try:
            url = url_str % (ticker, start, end, interval, crumb)
            hist = io.StringIO(requests.get(url, cookies={'B': cookie}).text)
            dfs[ticker] = parse_ticker_csv(hist, auto_adjust)
            if progress:
                pbar.animate()
        except:
            # something went wrong...
            # try one more time using a new cookie/crumb
            if not tried_once:
                tried_once = True
                try:
                    crumb, cookie = get_yahoo_crumb(force=True)
                    url = url_str % (ticker, start, end, interval, crumb)
                    src = requests.get(url, cookies={'B': cookie})
                    hist = io.StringIO(src.text)
                    dfs[ticker] = parse_ticker_csv(hist, auto_adjust)
                    if progress:
                        pbar.animate()
                except:
                    round1_failed_tickers.append(ticker)
        time.sleep(0.000001)

    # try failed items again before giving up
    if len(round1_failed_tickers) > 0:
        crumb, cookie = get_yahoo_crumb(force=True)
        for ticker in round1_failed_tickers:
            try:
                url = url_str % (ticker, start, end, interval, crumb)
                src = requests.get(url, cookies={'B': cookie})
                hist = io.StringIO(src.text)
                dfs[ticker] = parse_ticker_csv(hist, auto_adjust)
                if progress:
                    pbar.animate()
            except:
                round2_failed_tickers.append(ticker)
                pass
            time.sleep(0.000001)

        # if len(round2_failed_tickers) > 0:
        #     print("\nThe following tickers failed to download:\n",
        #           ', '.join(round2_failed_tickers))

    # create pandl (derecated)
    if as_panel:
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", category=DeprecationWarning)
            data = pd.Panel(dfs)
            if group_by == 'column':
                data = data.swapaxes(0, 2)

    # create multiIndex df
    else:
        data = pd.concat(dfs.values(), axis=1, keys=dfs.keys())
        if group_by == 'column':
            data.columns = data.columns.swaplevel(0, 1)
            data.sort_index(level=0, axis=1, inplace=True)
            if auto_adjust:
                data = data[['Open', 'High', 'Low', 'Close', 'Volume']]
            else:
                data = data[['Open', 'High', 'Low',
                             'Close', 'Adj Close', 'Volume']]

    # return single df if only one ticker
    if len(tickers) == 1:
        data = dfs[tickers[0]]

    return data

class ProgressBar:
    def __init__(self, iterations, text='completed'):
        self.text = text
        self.iterations = iterations
        self.prog_bar = '[]'
        self.fill_char = '*'
        self.width = 50
        self.__update_amount(0)
        self.elapsed = 1

    def animate(self, iteration=None):
        if iteration is None:
            self.elapsed += 1
            iteration = self.elapsed
        else:
            self.elapsed += iteration

        # print('\r' + str(self), end='')
        # sys.stdout.flush()
        self.update_iteration()

    def update_iteration(self):
        self.__update_amount((self.elapsed / float(self.iterations)) * 100.0)
        self.prog_bar += '  %s of %s %s' % (
            self.elapsed, self.iterations, self.text)

    def __update_amount(self, new_amount):
        percent_done = int(round((new_amount / 100.0) * 100.0))
        all_full = self.width - 2
        num_hashes = int(round((percent_done / 100.0) * all_full))
        self.prog_bar = '[' + self.fill_char * \
            num_hashes + ' ' * (all_full - num_hashes) + ']'
        pct_place = (len(self.prog_bar) // 2) - len(str(percent_done))
        pct_string = '%d%%' % percent_done
        self.prog_bar = self.prog_bar[0:pct_place] + \
            (pct_string + self.prog_bar[pct_place + len(pct_string):])

    def __str__(self):
        return str(self.prog_bar)

# make pandas datareader optional
# otherwise can be called via fix_yahoo_finance.download(...)
try:
    import pandas_datareader
    pandas_datareader.data.get_data_yahoo = download
except:
    pass<commit_msg>fix duplicate column issue<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Yahoo! Finance Fix for Pandas Datareader
# https://github.com/ranaroussi/fix-yahoo-finance
#
# Copyright 2017 Ran Aroussi
#
# Licensed under the GNU Lesser General Public License, v3.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.gnu.org/licenses/lgpl-3.0.en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.0.8"
__author__ = "Ran Aroussi"
__all__ = ['download', 'get_yahoo_crumb', 'parse_ticker_csv']

import datetime
import numpy as np
import pandas as pd
import time
import io
import requests
import re
import warnings
import sys

_YAHOO_COOKIE_ = ''
_YAHOO_CRUMB_ = ''
_YAHOO_CHECKED_ = None
_YAHOO_TTL_ = 180

def get_yahoo_crumb(force=False):
    global _YAHOO_COOKIE_, _YAHOO_CRUMB_, _YAHOO_CHECKED_, _YAHOO_TTL_

    # use same cookie for 5 min
    if _YAHOO_CHECKED_ and not force:
        now = datetime.datetime.now()
        delta = (now - _YAHOO_CHECKED_).total_seconds()
        if delta < _YAHOO_TTL_:
            return (_YAHOO_CRUMB_, _YAHOO_COOKIE_)

    res = requests.get('https://finance.yahoo.com/quote/SPY/history')
    _YAHOO_COOKIE_ = res.cookies['B']

    pattern = re.compile('.*"CrumbStore":\{"crumb":"(?P<crumb>[^"]+)"\}')
    for line in res.text.splitlines():
        m = pattern.match(line)
        if m is not None:
            _YAHOO_CRUMB_ = m.groupdict()['crumb']

    # set global params
    _YAHOO_CHECKED_ = datetime.datetime.now()

    return (_YAHOO_CRUMB_, _YAHOO_COOKIE_)

def parse_ticker_csv(csv_str, auto_adjust):
    df = pd.read_csv(csv_str, index_col=0, error_bad_lines=False, warn_bad_lines=False
                     ).replace('null', np.nan).dropna()

    df.index = pd.to_datetime(df.index)
    df = df.apply(pd.to_numeric)
    df['Volume'] = df['Volume'].fillna(0).astype(int)

    if auto_adjust:
        ratio = df["Close"] / df["Adj Close"]
        df["Adj Open"] = df["Open"] / ratio
        df["Adj High"] = df["High"] / ratio
        df["Adj Low"] = df["Low"] / ratio

        df.drop(
            ["Open", "High", "Low", "Close"],
            axis=1, inplace=True)

        df.rename(columns={
            "Adj Open": "Open", "Adj High": "High",
            "Adj Low": "Low", "Adj Close": "Close"
        }, inplace=True)

        df = df[['Open', 'High', 'Low', 'Close', 'Volume']]

    return df.groupby(df.index).first()

def download(tickers, start=None, end=None, as_panel=True,
             group_by='column', auto_adjust=False, progress=True,
             *args, **kwargs):

    # format start
    if start is None:
        start = int(time.mktime(time.strptime('1950-01-01', '%Y-%m-%d')))
    elif isinstance(start, datetime.datetime):
        start = int(time.mktime(start.timetuple()))
    else:
        start = int(time.mktime(time.strptime(str(start), '%Y-%m-%d')))

    # format end
    if end is None:
        end = int(time.mktime(datetime.datetime.now().timetuple()))
    elif isinstance(end, datetime.datetime):
        end = int(time.mktime(end.timetuple()))
    else:
        end = int(time.mktime(time.strptime(str(end), '%Y-%m-%d')))

    # iterval
    interval = kwargs["interval"] if "interval" in kwargs else "1d"

    # url template
    url_str = "https://query1.finance.yahoo.com/v7/finance/download/%s"
    url_str += "?period1=%s&period2=%s&interval=%s&events=history&crumb=%s"

    # dataframe collector
    dfs = {}

    # create ticker list
    tickers = tickers if isinstance(tickers, list) else [tickers]
    tickers = [x.upper() for x in tickers]

    # initiate progress bar
    if progress:
        pbar = ProgressBar(len(tickers), 'downloaded')

    # failed tickers collectors
    round1_failed_tickers = []
    round2_failed_tickers = []

    # start downloading
    for ticker in tickers:

        # yahoo crumb/cookie
        crumb, cookie = get_yahoo_crumb()

        tried_once = False
        try:
            url = url_str % (ticker, start, end, interval, crumb)
            hist = io.StringIO(requests.get(url, cookies={'B': cookie}).text)
            dfs[ticker] = parse_ticker_csv(hist, auto_adjust)
            if progress:
                pbar.animate()
        except:
            # something went wrong...
            # try one more time using a new cookie/crumb
            if not tried_once:
                tried_once = True
                try:
                    crumb, cookie = get_yahoo_crumb(force=True)
                    url = url_str % (ticker, start, end, interval, crumb)
                    src = requests.get(url, cookies={'B': cookie})
                    hist = io.StringIO(src.text)
                    dfs[ticker] = parse_ticker_csv(hist, auto_adjust)
                    if progress:
                        pbar.animate()
                except:
                    round1_failed_tickers.append(ticker)
        time.sleep(0.000001)

    # try failed items again before giving up
    if len(round1_failed_tickers) > 0:
        crumb, cookie = get_yahoo_crumb(force=True)
        for ticker in round1_failed_tickers:
            try:
                url = url_str % (ticker, start, end, interval, crumb)
                src = requests.get(url, cookies={'B': cookie})
                hist = io.StringIO(src.text)
                dfs[ticker] = parse_ticker_csv(hist, auto_adjust)
                if progress:
                    pbar.animate()
            except:
                round2_failed_tickers.append(ticker)
                pass
            time.sleep(0.000001)

        # if len(round2_failed_tickers) > 0:
        #     print("\nThe following tickers failed to download:\n",
        #           ', '.join(round2_failed_tickers))

    # create pandl (derecated)
    if as_panel:
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", category=DeprecationWarning)
            data = pd.Panel(dfs)
            if group_by == 'column':
                data = data.swapaxes(0, 2)

    # create multiIndex df
    else:
        data = pd.concat(dfs.values(), axis=1, keys=dfs.keys())
        if group_by == 'column':
            data.columns = data.columns.swaplevel(0, 1)
            data.sort_index(level=0, axis=1, inplace=True)
            if auto_adjust:
                data = data[['Open', 'High', 'Low', 'Close', 'Volume']]
            else:
                data = data[['Open', 'High', 'Low',
                             'Close', 'Adj Close', 'Volume']]

    # return single df if only one ticker
    if len(tickers) == 1:
        data = dfs[tickers[0]]

    return data

class ProgressBar:
    def __init__(self, iterations, text='completed'):
        self.text = text
        self.iterations = iterations
        self.prog_bar = '[]'
        self.fill_char = '*'
        self.width = 50
        self.__update_amount(0)
        self.elapsed = 1

    def animate(self, iteration=None):
        if iteration is None:
            self.elapsed += 1
            iteration = self.elapsed
        else:
            self.elapsed += iteration

        # print('\r' + str(self), end='')
        # sys.stdout.flush()
        self.update_iteration()

    def update_iteration(self):
        self.__update_amount((self.elapsed / float(self.iterations)) * 100.0)
        self.prog_bar += '  %s of %s %s' % (
            self.elapsed, self.iterations, self.text)

    def __update_amount(self, new_amount):
        percent_done = int(round((new_amount / 100.0) * 100.0))
        all_full = self.width - 2
        num_hashes = int(round((percent_done / 100.0) * all_full))
        self.prog_bar = '[' + self.fill_char * \
            num_hashes + ' ' * (all_full - num_hashes) + ']'
        pct_place = (len(self.prog_bar) // 2) - len(str(percent_done))
        pct_string = '%d%%' % percent_done
        self.prog_bar = self.prog_bar[0:pct_place] + \
            (pct_string + self.prog_bar[pct_place + len(pct_string):])

    def __str__(self):
        return str(self.prog_bar)

# make pandas datareader optional
# otherwise can be called via fix_yahoo_finance.download(...)
try:
    import pandas_datareader
    pandas_datareader.data.get_data_yahoo = download
except:
    pass<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Production example script that averages repeated tranects, resolving and visualizing secondary circulation
Driver script that is designed to average repeated ADCP transect observations
in an effort of reduce measurement error and better resolve non-steamwise and
velocities and secondary circulation features.  A summary of script functions:
    1) Assess an input directory for ADCP observations (raw files) that match
       in space and time.
    2) Group matchcing ADCP observations into groups of a maxium number for
       averaging.
    3) Pre-process raw ADCP observations before averaging as appropriate.
    4) Bin-average pre-processed ADCP observation velcotities
    5) Generate netcdf and/or CSV output files of bin-average velocities
    6) Generate various plots of streamwise, depth averaged, 3D velocities
       and secondary circulation features.

The script options are listed and described immediatly below this comment
block.

This code is open source, and defined by the included MIT Copyright License

Designed for Python 2.7; NumPy 1.7; SciPy 0.11.0; Matplotlib 1.2.0
2014-09 - First Release; blsaenz, esatel
"""
## START script options #######################################################

# bin-averaging paramterters
avg_dxy                        = 2.0          # Horizontal resolution of averaging bins {m}
avg_dz                         = 0.25         # Vertical resolution of averaging bins {m}
avg_max_gap_m                  = 30.0         # Maximum distance allowed between ADCP observations when averaging {m}
avg_max_gap_minutes            = 20.0         # Maximum time allowed between ADCP observations when averaging {m}
avg_max_group_size             = 6            # Maximum number of ADCP observations to average {m}

# post-average options
avg_rotation                   = 'Rozovski'   # One of ['Rozovski','no transverse flow','principal flow','normal',None]
avg_std_drop                   = 3.0          # Standard deviation of velocity, above which samples are dropped from analysis {0.0=no dropping, 2.0-3.0 typically [number of standard deviations]}
avg_std_interp                 = True         # Perform interpolation of holes in velocity profiles left by high standard deviation removal {typically True with std_drop > 0.0}
avg_smooth_kernel              = 5            # Smooth velocity data using a square kernel box-filter, with side dimension = avg_smooth_kernel.  0 = no kernel smoothing
avg_save_netcdf                = True         # Save bin-averaged velocities as an ADCPData netcdf file
avg_save_csv                   = True         # Save bin-averaged velocities as a CSV text file
avg_plot_xy                    = True         # Generate a composite plot of survey location(s) of original ADCP ensembles
avg_plot_avg_n_sd              = True         # Generate image plots of bin-averaged U,V,W velocities, and the number and standard deviation of bin averages
avg_plot_mean_vectors          = True         # Generate an arrow plot of bin-averaged U-V mean velocities in the x-y plane
avg_plot_secondary_circulation = True         # Generate an image plot of 2D bin-averaged steamwise (u) velocities, overlain by an arrow plot showing secondary circulation in the V-W plane
avg_plot_uvw_velocity_array    = True         # Generate a 3-panel image plot showing bin-averaged U,V,W velocities in the V-W plane
avg_plot_flow_summmary         = True         # Generate a summary plot showing image plots of U,V bin-averaged velocities, an arrow plot of bin-averaged U-V mean velocities, and flow/discharge calculations
avg_save_plots                 = True         # Save the plots to disk
avg_show_plots                 = False        # Print plots to screen (pauses execution until plots are manually closed)

## END script options #########################################################

import os
import numpy as np
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
import transect_preprocessor
reload(transect_preprocessor)
import adcpy
reload(adcpy)
from adcpy_recipes import *

def transect_average(pre_process_input_file=None):
    """
    Received a list of ADCPTransectData objects from transect_preprocessor.py,
    and then groups and bin-averages the transects.  Group average ADCPData
    objects, velocties, and plots are optionally output to the outpath supplied
    by transect_preprocessor().
    Inputs:
        pre_process_input_file = path to a transect_preprocessor input file,
          or None to use the default file.
    """

    (transects,outpath) = transect_preprocessor.transect_preprocessor(pre_process_input_file)
    print 'total transects loaded:',len(transects)
    grps_to_average = group_adcp_obs_by_spacetime(transects,
                                                  max_gap_m=avg_max_gap_m,
                                                  max_gap_minutes=avg_max_gap_minutes,
                                                  max_group_size=avg_max_group_size)
    print 'total groups to average',len(grps_to_average)
    #write_csv_velocity(transects[0],os.path.join(outpath,'temp.csv'),no_header=True)
    #grps_to_average = grps_to_average[1:]

    grp_num = 0
    track_fig=101
    for grp in grps_to_average:
        if avg_plot_xy:
            adcpy.plot.plot_obs_group_xy_lines(grp,fig=track_fig,title='Group%03i Source Observations'%grp_num)

        avg = average_transects(grp,dxy=avg_dxy,dz=avg_dz,return_adcpy=True,
                                plotline_from_flow=True)

        if avg_plot_xy:
            adcpy.plot.get_fig(fig=track_fig)
            plt.plot(avg.xy[:,0],avg.xy[:,1],label='average projection')
            plt.legend(prop={'size':10})
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_xy_lines.png"%grp_num))

        if avg_rotation is not None:
            avg = transect_rotate(avg,avg_rotation)
        if avg_std_drop > 0:
            avg.sd_drop(sd=3.0,
                      sd_axis='elevation',
                      interp_holes=avg_std_interp)
            avg.sd_drop(sd=3.0,
                      sd_axis='ensemble',
                      interp_holes=avg_std_interp)
        if avg_smooth_kernel > 2:
            avg.kernel_smooth(kernel_size = 3)

        if avg_save_csv:
            write_csv_velocity(avg,os.path.join(outpath,'group%03i_velocity.csv'%grp_num),no_header=True)
        if avg_save_netcdf:
            fname = os.path.join(outpath,'group%03i.nc'%grp_num)
            avg.write_nc(fname,zlib=True)

        if avg_plot_avg_n_sd:
            uvw = 'uvw'
            for i in range(3):
                plot_avg_n_sd(avg,i,0.05)
                if avg_save_plots:
                    plt.savefig(os.path.join(outpath,"group%03i_%s_avg_n_sd.png"%(grp_num,uvw[i])))

        if avg_plot_mean_vectors:
            fig3 = adcpy.plot.plot_ensemble_mean_vectors(avg,title='Group%03i Mean Velocity [m/s]'%grp_num)
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_mean_velocity.png"%grp_num))

        if avg_plot_secondary_circulation:
            fig4 = adcpy.plot.plot_secondary_circulation(avg,u_vecs=30,v_vecs=30,
                                                         title='Group%03i Streamwise Velocity [m/s] and Cross-stream Vectors'%grp_num)
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_secondary_circulation.png"%grp_num))

        if avg_plot_uvw_velocity_array:
            fig5 = adcpy.plot.plot_uvw_velocity_array(avg.velocity,
                                                      title='Group%03i Velocity [m/s]'%grp_num,
                                                      ures=0.1,vres=0.1,wres=0.05)
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_uvw_velocity.png"%grp_num))

        if avg_plot_flow_summmary:
            fig6 = adcpy.plot.plot_flow_summmary(avg,title='Group%03i Streawise Summary'%grp_num,
                                                 ures=0.1,vres=0.1,use_grid_flows=True)
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_flow_summary.png"%grp_num))

        if avg_show_plots:
            plt.show()
        plt.close('all')
        grp_num += 1

    print 'ADCP processing complete!'

def plot_avg_n_sd(avg,uvw,resolution=0.1):
    """
    Generates a vertical three-panel plot, showing images of a bin-averaged
    velocity, the number of velociy measurements in each bin, and the bin standard
    deviation velocity.  Desinged to be used in conjuction with
    transect_average() output.
    Inputs:
        avg = ADCPData object, with extra velocity_n and velocity_sd data
          fields produced by transect_average()
        uvw = python string, either 'U','V', or 'W' to select which velocity
          compontent to plot.
        resolution = optional value to round the plot velocity scales up toward
    """

    if uvw == 0:
        v_str = 'U'
    elif uvw == 1:
        v_str = 'V'
    else:
        v_str = 'W'

    inv = 1/resolution
    xx,yy,dd,pline = adcpy.util.find_projection_distances(avg.xy)
    mtest = np.floor(avg.velocity[...,uvw]*inv)
    minv = np.nanmin(np.nanmin(mtest))*resolution
    mtest = np.ceil(avg.velocity[...,uvw]*inv)
    maxv = np.nanmax(np.nanmax(mtest))*resolution
    avg_panel = adcpy.plot.IPanel(velocity = avg.velocity[:,:,uvw],
                             x = dd,
                             y = avg.bin_center_elevation,
                             minv = minv,
                             maxv = maxv,
                             xlabel = 'm',
                             ylabel = 'm',
                             units = 'm/s',
                             use_pcolormesh = True,
                             title='%s Averaged Velocity [m/s]'%v_str)
    maxv = np.nanmax(np.nanmax(avg.velocity_n[...,uvw]))
    n_panel = adcpy.plot.IPanel(velocity = avg.velocity_n[:,:,uvw],
                             x = dd,
                             y = avg.bin_center_elevation,
                             minv = 0,
                             maxv = maxv,
                             xlabel = 'm',
                             ylabel = 'm',
                             units = 'number',
                             use_pcolormesh = True,
                             title='n Samples')
    mtest = np.floor(avg.velocity_sd[...,uvw]*inv)
    minv = np.nanmin(np.nanmin(mtest))*resolution
    mtest = np.ceil(avg.velocity_sd[...,uvw]*inv)
    maxv = np.nanmax(np.nanmax(mtest))*resolution
    sd_panel = adcpy.plot.IPanel(velocity = avg.velocity_sd[:,:,uvw],
                             x = dd,
                             y = avg.bin_center_elevation,
                             minv = 0,
                             maxv = maxv,
                             xlabel = 'm',
                             ylabel = 'm',
                             units = 'm/s',
                             use_pcolormesh = True,
                             title='Standard Deviation [m/s]')
    fig = adcpy.plot.plot_vertical_panels((avg_panel,n_panel,sd_panel))
    return fig

def main():
    #prepro_input = sys.argv[1]
    transect_average(r'trn_pre_input_GEO20090106.py')
    #transect_average()

# run myself
if __name__ == "__main__":
    main()

<commit_msg>Reinstated use of sys.argv<commit_after># -*- coding: utf-8 -*-
"""Production example script that averages repeated tranects, resolving and visualizing secondary circulation
Driver script that is designed to average repeated ADCP transect observations
in an effort of reduce measurement error and better resolve non-steamwise and
velocities and secondary circulation features.  A summary of script functions:
    1) Assess an input directory for ADCP observations (raw files) that match
       in space and time.
    2) Group matchcing ADCP observations into groups of a maxium number for
       averaging.
    3) Pre-process raw ADCP observations before averaging as appropriate.
    4) Bin-average pre-processed ADCP observation velcotities
    5) Generate netcdf and/or CSV output files of bin-average velocities
    6) Generate various plots of streamwise, depth averaged, 3D velocities
       and secondary circulation features.

The script options are listed and described immediatly below this comment
block.

This code is open source, and defined by the included MIT Copyright License

Designed for Python 2.7; NumPy 1.7; SciPy 0.11.0; Matplotlib 1.2.0
2014-09 - First Release; blsaenz, esatel
"""
## START script options #######################################################

# bin-averaging paramterters
avg_dxy                        = 2.0          # Horizontal resolution of averaging bins {m}
avg_dz                         = 0.25         # Vertical resolution of averaging bins {m}
avg_max_gap_m                  = 30.0         # Maximum distance allowed between ADCP observations when averaging {m}
avg_max_gap_minutes            = 20.0         # Maximum time allowed between ADCP observations when averaging {m}
avg_max_group_size             = 6            # Maximum number of ADCP observations to average {m}

# post-average options
avg_rotation                   = 'Rozovski'   # One of ['Rozovski','no transverse flow','principal flow','normal',None]
avg_std_drop                   = 3.0          # Standard deviation of velocity, above which samples are dropped from analysis {0.0=no dropping, 2.0-3.0 typically [number of standard deviations]}
avg_std_interp                 = True         # Perform interpolation of holes in velocity profiles left by high standard deviation removal {typically True with std_drop > 0.0}
avg_smooth_kernel              = 5            # Smooth velocity data using a square kernel box-filter, with side dimension = avg_smooth_kernel.  0 = no kernel smoothing
avg_save_netcdf                = True         # Save bin-averaged velocities as an ADCPData netcdf file
avg_save_csv                   = True         # Save bin-averaged velocities as a CSV text file
avg_plot_xy                    = True         # Generate a composite plot of survey location(s) of original ADCP ensembles
avg_plot_avg_n_sd              = True         # Generate image plots of bin-averaged U,V,W velocities, and the number and standard deviation of bin averages
avg_plot_mean_vectors          = True         # Generate an arrow plot of bin-averaged U-V mean velocities in the x-y plane
avg_plot_secondary_circulation = True         # Generate an image plot of 2D bin-averaged steamwise (u) velocities, overlain by an arrow plot showing secondary circulation in the V-W plane
avg_plot_uvw_velocity_array    = True         # Generate a 3-panel image plot showing bin-averaged U,V,W velocities in the V-W plane
avg_plot_flow_summmary         = True         # Generate a summary plot showing image plots of U,V bin-averaged velocities, an arrow plot of bin-averaged U-V mean velocities, and flow/discharge calculations
avg_save_plots                 = True         # Save the plots to disk
avg_show_plots                 = False        # Print plots to screen (pauses execution until plots are manually closed)

## END script options #########################################################

import os
import numpy as np
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
import transect_preprocessor
reload(transect_preprocessor)
import adcpy
reload(adcpy)
from adcpy_recipes import *

def transect_average(pre_process_input_file=None):
    """
    Received a list of ADCPTransectData objects from transect_preprocessor.py,
    and then groups and bin-averages the transects.  Group average ADCPData
    objects, velocties, and plots are optionally output to the outpath supplied
    by transect_preprocessor().
    Inputs:
        pre_process_input_file = path to a transect_preprocessor input file,
          or None to use the default file.
    """

    (transects,outpath) = transect_preprocessor.transect_preprocessor(pre_process_input_file)
    print 'total transects loaded:',len(transects)
    grps_to_average = group_adcp_obs_by_spacetime(transects,
                                                  max_gap_m=avg_max_gap_m,
                                                  max_gap_minutes=avg_max_gap_minutes,
                                                  max_group_size=avg_max_group_size)
    print 'total groups to average',len(grps_to_average)
    #write_csv_velocity(transects[0],os.path.join(outpath,'temp.csv'),no_header=True)
    #grps_to_average = grps_to_average[1:]

    grp_num = 0
    track_fig=101
    for grp in grps_to_average:
        if avg_plot_xy:
            adcpy.plot.plot_obs_group_xy_lines(grp,fig=track_fig,title='Group%03i Source Observations'%grp_num)

        avg = average_transects(grp,dxy=avg_dxy,dz=avg_dz,return_adcpy=True,
                                plotline_from_flow=True)

        if avg_plot_xy:
            adcpy.plot.get_fig(fig=track_fig)
            plt.plot(avg.xy[:,0],avg.xy[:,1],label='average projection')
            plt.legend(prop={'size':10})
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_xy_lines.png"%grp_num))

        if avg_rotation is not None:
            avg = transect_rotate(avg,avg_rotation)
        if avg_std_drop > 0:
            avg.sd_drop(sd=3.0,
                      sd_axis='elevation',
                      interp_holes=avg_std_interp)
            avg.sd_drop(sd=3.0,
                      sd_axis='ensemble',
                      interp_holes=avg_std_interp)
        if avg_smooth_kernel > 2:
            avg.kernel_smooth(kernel_size = 3)

        if avg_save_csv:
            write_csv_velocity(avg,os.path.join(outpath,'group%03i_velocity.csv'%grp_num),no_header=True)
        if avg_save_netcdf:
            fname = os.path.join(outpath,'group%03i.nc'%grp_num)
            avg.write_nc(fname,zlib=True)

        if avg_plot_avg_n_sd:
            uvw = 'uvw'
            for i in range(3):
                plot_avg_n_sd(avg,i,0.05)
                if avg_save_plots:
                    plt.savefig(os.path.join(outpath,"group%03i_%s_avg_n_sd.png"%(grp_num,uvw[i])))

        if avg_plot_mean_vectors:
            fig3 = adcpy.plot.plot_ensemble_mean_vectors(avg,title='Group%03i Mean Velocity [m/s]'%grp_num)
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_mean_velocity.png"%grp_num))

        if avg_plot_secondary_circulation:
            fig4 = adcpy.plot.plot_secondary_circulation(avg,u_vecs=30,v_vecs=30,
                                                         title='Group%03i Streamwise Velocity [m/s] and Cross-stream Vectors'%grp_num)
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_secondary_circulation.png"%grp_num))

        if avg_plot_uvw_velocity_array:
            fig5 = adcpy.plot.plot_uvw_velocity_array(avg.velocity,
                                                      title='Group%03i Velocity [m/s]'%grp_num,
                                                      ures=0.1,vres=0.1,wres=0.05)
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_uvw_velocity.png"%grp_num))

        if avg_plot_flow_summmary:
            fig6 = adcpy.plot.plot_flow_summmary(avg,title='Group%03i Streawise Summary'%grp_num,
                                                 ures=0.1,vres=0.1,use_grid_flows=True)
            if avg_save_plots:
                plt.savefig(os.path.join(outpath,"group%03i_flow_summary.png"%grp_num))

        if avg_show_plots:
            plt.show()
        plt.close('all')
        grp_num += 1

    print 'ADCP processing complete!'

def plot_avg_n_sd(avg,uvw,resolution=0.1):
    """
    Generates a vertical three-panel plot, showing images of a bin-averaged
    velocity, the number of velociy measurements in each bin, and the bin standard
    deviation velocity.  Desinged to be used in conjuction with
    transect_average() output.
    Inputs:
        avg = ADCPData object, with extra velocity_n and velocity_sd data
          fields produced by transect_average()
        uvw = python string, either 'U','V', or 'W' to select which velocity
          compontent to plot.
        resolution = optional value to round the plot velocity scales up toward
    """

    if uvw == 0:
        v_str = 'U'
    elif uvw == 1:
        v_str = 'V'
    else:
        v_str = 'W'

    inv = 1/resolution
    xx,yy,dd,pline = adcpy.util.find_projection_distances(avg.xy)
    mtest = np.floor(avg.velocity[...,uvw]*inv)
    minv = np.nanmin(np.nanmin(mtest))*resolution
    mtest = np.ceil(avg.velocity[...,uvw]*inv)
    maxv = np.nanmax(np.nanmax(mtest))*resolution
    avg_panel = adcpy.plot.IPanel(velocity = avg.velocity[:,:,uvw],
                             x = dd,
                             y = avg.bin_center_elevation,
                             minv = minv,
                             maxv = maxv,
                             xlabel = 'm',
                             ylabel = 'm',
                             units = 'm/s',
                             use_pcolormesh = True,
                             title='%s Averaged Velocity [m/s]'%v_str)
    maxv = np.nanmax(np.nanmax(avg.velocity_n[...,uvw]))
    n_panel = adcpy.plot.IPanel(velocity = avg.velocity_n[:,:,uvw],
                             x = dd,
                             y = avg.bin_center_elevation,
                             minv = 0,
                             maxv = maxv,
                             xlabel = 'm',
                             ylabel = 'm',
                             units = 'number',
                             use_pcolormesh = True,
                             title='n Samples')
    mtest = np.floor(avg.velocity_sd[...,uvw]*inv)
    minv = np.nanmin(np.nanmin(mtest))*resolution
    mtest = np.ceil(avg.velocity_sd[...,uvw]*inv)
    maxv = np.nanmax(np.nanmax(mtest))*resolution
    sd_panel = adcpy.plot.IPanel(velocity = avg.velocity_sd[:,:,uvw],
                             x = dd,
                             y = avg.bin_center_elevation,
                             minv = 0,
                             maxv = maxv,
                             xlabel = 'm',
                             ylabel = 'm',
                             units = 'm/s',
                             use_pcolormesh = True,
                             title='Standard Deviation [m/s]')
    fig = adcpy.plot.plot_vertical_panels((avg_panel,n_panel,sd_panel))
    return fig

def main():
    import sys
    prepro_input = sys.argv[1]
    transect_average(prepro_input)

# run myself
if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># coding:utf-8

import os
import base64
import configparser
import json
import urllib

import pandas as pd
import requests
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes

from QUANTAXIS.QAMarket.QABroker import QA_Broker
from QUANTAXIS.QAUtil.QASetting import setting_path

CONFIGFILE_PATH = '{}{}{}'.format(setting_path, os.sep, 'config.ini')
DEFAULT_SHIPANE_URL = 'http://127.0.0.1:8888'

def get_config_SPE():
    config = configparser.ConfigParser()
    if os.path.exists(CONFIGFILE_PATH):
        config.read(CONFIGFILE_PATH)
        try:
            return config.get('SPE', 'uri')
        except configparser.NoSectionError:
            config.add_section('SPE')
            config.set('SPE', 'uri', DEFAULT_SHIPANE_URL)
            return DEFAULT_SHIPANE_URL
        except configparser.NoOptionError:
            config.set('SPE', 'uri', DEFAULT_SHIPANE_URL)
            return DEFAULT_SHIPANE_URL
        finally:

            with open(CONFIGFILE_PATH, 'w') as f:
                config.write(f)

    else:
        f = open(CONFIGFILE_PATH, 'w')
        config.add_section('SPE')
        config.set('SPE', 'uri', DEFAULT_SHIPANE_URL)
        config.write(f)
        f.close()
        return DEFAULT_SHIPANE_URL

class SPETradeApi(QA_Broker):
    def __init__(self, endpoint=get_config_SPE()):

        self._endpoint = endpoint
        self._session = requests
        self.fillorder_headers = ['name', 'datetime', 'towards', 'price',
                                  'amount', 'money', 'trade_id', 'order_id', 'code', 'shareholder', 'other']
        self.holding_headers = ['code', 'name', 'hoding_price', 'price', 'pnl', 'amount',
                                'sell_available', 'pnl_money', 'holdings', 'total_amount', 'lastest_amounts', 'shareholder']
        self.askorder_headers = ['code', 'towards', 'price', 'amount', 'transaction_price',
                                 'transaction_amount', 'status', 'order_time', 'order_id', 'id', 'code', 'shareholders']

    def call(self, func, params=''):
        response = self._session.get(
            '{}/api/v1.0/{}'.format(self._endpoint, func), params)

        text = response.text

        return json.loads(text)

    def call_post(self, func, params={}):
        uri = '{}/api/v1.0/{}?client={}'.format(
            self._endpoint, func, params.pop('client'))
        response = self._session.post(uri, json=params)
        text = response.text
        return json.loads(text)

    def call_delete(self, func, params=''):
        response = self._session.delete(
            '{}/api/v1.0/{}'.format(self._endpoint, func))

        text = response.text
        print(text)
        try:
            return json.loads(text)
        except:
            return text

    def data_to_df(self, result):
        return pd.DataFrame(data=result)

    #------ functions

    def ping(self):
        return self.call("ping", {})

    def query_accounts(self, accounts):
        return self.call("accounts", {
            'client': accounts
        })

    def query_positions(self, accounts):
        return self.call("positions", {
            'client': accounts
        })

    def query_orders(self, accounts, status='filled'):
        return self.call("orders", {
            'client': accounts,
            'status': status
        })

    def send_order(self, accounts):
        return self.call_post('orders', {
            'client': accounts,
            "action": "BUY",
            "symbol": "000001",
            "type": "LIMIT",
            "priceType": 0,
            "price": 9.26,
            "amount": 100

        })

    def cancel_order(self, accounts, orderid):
        return self.call_delete('orders/{}'.format(orderid), json.dumps({
            'client': accounts
        }))

    def cancel_all(self):
        return self.call_delete('orders')

if __name__ == '__main__':
    a = SPETradeApi()
    a.query_accounts('account:1363350141')
    a.query_orders('account:141')
    a.query_orders('account:141', 'open')
    a.send_order('account:141')
    a.cancel_all()

    # a.cancel_order('account:141','919')
<commit_msg>#update<commit_after># coding:utf-8

import os
import base64
import configparser
import json
import urllib

import pandas as pd
import requests
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes

from QUANTAXIS.QAMarket.QABroker import QA_Broker
from QUANTAXIS.QAUtil.QASetting import setting_path

CONFIGFILE_PATH = '{}{}{}'.format(setting_path, os.sep, 'config.ini')
DEFAULT_SHIPANE_URL = 'http://127.0.0.1:8888'

def get_config_SPE():
    config = configparser.ConfigParser()
    if os.path.exists(CONFIGFILE_PATH):
        config.read(CONFIGFILE_PATH)
        try:
            return config.get('SPE', 'uri')
        except configparser.NoSectionError:
            config.add_section('SPE')
            config.set('SPE', 'uri', DEFAULT_SHIPANE_URL)
            return DEFAULT_SHIPANE_URL
        except configparser.NoOptionError:
            config.set('SPE', 'uri', DEFAULT_SHIPANE_URL)
            return DEFAULT_SHIPANE_URL
        finally:

            with open(CONFIGFILE_PATH, 'w') as f:
                config.write(f)

    else:
        f = open(CONFIGFILE_PATH, 'w')
        config.add_section('SPE')
        config.set('SPE', 'uri', DEFAULT_SHIPANE_URL)
        config.write(f)
        f.close()
        return DEFAULT_SHIPANE_URL

class SPETradeApi(QA_Broker):
    def __init__(self, endpoint=get_config_SPE()):

        self._endpoint = endpoint
        self._session = requests
        self.fillorder_headers = ['name', 'datetime', 'towards', 'price',
                                  'amount', 'money', 'trade_id', 'order_id', 'code', 'shareholder', 'other']
        self.holding_headers = ['code', 'name', 'hoding_price', 'price', 'pnl', 'amount',
                                'sell_available', 'pnl_money', 'holdings', 'total_amount', 'lastest_amounts', 'shareholder']
        self.askorder_headers = ['code', 'towards', 'price', 'amount', 'transaction_price',
                                 'transaction_amount', 'status', 'order_time', 'order_id', 'id', 'code', 'shareholders']

    def call(self, func, params=''):
        response = self._session.get(
            '{}/api/v1.0/{}'.format(self._endpoint, func), params)

        text = response.text

        return json.loads(text)

    def call_post(self, func, params={}):
        uri = '{}/api/v1.0/{}?client={}'.format(
            self._endpoint, func, params.pop('client'))
        response = self._session.post(uri, json=params)
        text = response.text
        return json.loads(text)

    def call_delete(self, func, params=''):
        response = self._session.delete(
            '{}/api/v1.0/{}'.format(self._endpoint, func))

        text = response.text
        print(text)
        try:
            return json.loads(text)
        except:
            return text

    def data_to_df(self, result):
        return pd.DataFrame(data=result)

    #------ functions

    def ping(self):
        return self.call("ping", {})

    def query_accounts(self, accounts):
        return self.call("accounts", {
            'client': accounts
        })

    def query_positions(self, accounts):
        return self.call("positions", {
            'client': accounts
        })
    def query_clients(self):
        return self.call("clients")
    def query_orders(self, accounts, status='filled'):
        return self.call("orders", {
            'client': accounts,
            'status': status
        })

    def send_order(self, accounts):
        return self.call_post('orders', {
            'client': accounts,
            "action": "BUY",
            "symbol": "000001",
            "type": "LIMIT",
            "priceType": 0,
            "price": 9.26,
            "amount": 100

        })

    def cancel_order(self, accounts, orderid):
        return self.call_delete('orders/{}'.format(orderid), json.dumps({
            'client': accounts
        }))

    def cancel_all(self):
        return self.call_delete('orders')

if __name__ == '__main__':
    a = SPETradeApi()
    a.query_accounts('account:1363350141')
    a.query_orders('account:141')
    a.query_orders('account:141', 'open')
    a.send_order('account:141')
    a.cancel_all()

    # a.cancel_order('account:141','919')
<|endoftext|>


<commit_before>#!/usr/bin/env python

## \file parallel_regression.py
#  \brief Python script for automated regression testing of SU2 examples
#  \author A. Aranake, A. Campos, T. Economon, T. Lukaczyk, S. Padron
#  \version 5.0.0 "Raven"
#
# SU2 Original Developers: Dr. Francisco D. Palacios.
#                          Dr. Thomas D. Economon.
#
# SU2 Developers: Prof. Juan J. Alonso's group at Stanford University.
#                 Prof. Piero Colonna's group at Delft University of Technology.
#                 Prof. Nicolas R. Gauger's group at Kaiserslautern University of Technology.
#                 Prof. Alberto Guardone's group at Polytechnic University of Milan.
#                 Prof. Rafael Palacios' group at Imperial College London.
#                 Prof. Edwin van der Weide's group at the University of Twente.
#                 Prof. Vincent Terrapon's group at the University of Liege.
#
# Copyright (C) 2012-2017 SU2, the open-source CFD code.
#
# SU2 is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# SU2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with SU2. If not, see <http://www.gnu.org/licenses/>.

import sys
from TestCase import TestCase

def main():
    '''This program runs SU2 and ensures that the output matches specified values.
       This will be used to do checks when code is pushed to github
       to make sure nothing is broken. '''

    test_list = []

    #####################################
    ### Disc. adj. compressible Euler ###
    #####################################

    # Inviscid NACA0012
    discadj_naca0012           = TestCase('discadj_naca0012')
    discadj_naca0012.cfg_dir   = "cont_adj_euler/naca0012"
    discadj_naca0012.cfg_file  = "inv_NACA0012_discadj.cfg"
    discadj_naca0012.test_iter = 100
    discadj_naca0012.test_vals = [-3.606963, -9.034619, -0.000000, 0.005688] #last 4 columns
    discadj_naca0012.su2_exec  = "parallel_computation.py -f"
    discadj_naca0012.timeout   = 1600
    discadj_naca0012.tol       = 0.00001
    test_list.append(discadj_naca0012)

    ####################################
    ### Disc. adj. compressible RANS ###
    ####################################

    # Adjoint turbulent NACA0012 SA
    discadj_rans_naca0012_sa           = TestCase('discadj_rans_naca0012_sa')
    discadj_rans_naca0012_sa.cfg_dir   = "disc_adj_rans/naca0012"
    discadj_rans_naca0012_sa.cfg_file  = "turb_NACA0012_sa.cfg"
    discadj_rans_naca0012_sa.test_iter = 10
    discadj_rans_naca0012_sa.test_vals = [-1.751997, 0.485368, 0.182890, -0.000018] #last 4 columns
    discadj_rans_naca0012_sa.su2_exec  = "parallel_computation.py -f"
    discadj_rans_naca0012_sa.timeout   = 1600
    discadj_rans_naca0012_sa.tol       = 0.00001
    test_list.append(discadj_rans_naca0012_sa)

    # Adjoint turbulent NACA0012 SST
    discadj_rans_naca0012_sst           = TestCase('discadj_rans_naca0012_sst')
    discadj_rans_naca0012_sst.cfg_dir   = "disc_adj_rans/naca0012"
    discadj_rans_naca0012_sst.cfg_file  = "turb_NACA0012_sst.cfg"
    discadj_rans_naca0012_sst.test_iter = 10
    discadj_rans_naca0012_sst.test_vals = [-1.655243, -0.507187, 0.129010, -0.000013] #last 4 columns
    discadj_rans_naca0012_sst.su2_exec  = "parallel_computation.py -f"
    discadj_rans_naca0012_sst.timeout   = 1600
    discadj_rans_naca0012_sst.tol       = 0.00001
    test_list.append(discadj_rans_naca0012_sst)

    #######################################################
    ### Unsteady Disc. adj. compressible RANS           ###
    #######################################################

    # Turbulent Cylinder
    discadj_cylinder           = TestCase('unsteady_cylinder')
    discadj_cylinder.cfg_dir   = "disc_adj_rans/cylinder"
    discadj_cylinder.cfg_file  = "cylinder.cfg"
    discadj_cylinder.test_iter = 10
    discadj_cylinder.test_vals = [3.522085, -1.787791, -0.012031, 0.000017] #last 4 columns
    discadj_cylinder.su2_exec  = "parallel_computation.py -f"
    discadj_cylinder.timeout   = 1600
    discadj_cylinder.tol       = 0.00001
    discadj_cylinder.unsteady  = True
    test_list.append(discadj_cylinder
)
    ######################################
    ### RUN TESTS                      ###
    ######################################

    pass_list = [ test.run_test() for test in test_list ]

    # Tests summary
    print '=================================================================='
    print 'Summary of the parallel tests'
    for i, test in enumerate(test_list):
        if (pass_list[i]):
            print '  passed - %s'%test.tag
        else:
            print '* FAILED - %s'%test.tag

    if all(pass_list):
        sys.exit(0)
    else:
        sys.exit(1)
    # done

if __name__ == '__main__':
    main()
<commit_msg>Update of reg. test values<commit_after>#!/usr/bin/env python

## \file parallel_regression.py
#  \brief Python script for automated regression testing of SU2 examples
#  \author A. Aranake, A. Campos, T. Economon, T. Lukaczyk, S. Padron
#  \version 5.0.0 "Raven"
#
# SU2 Original Developers: Dr. Francisco D. Palacios.
#                          Dr. Thomas D. Economon.
#
# SU2 Developers: Prof. Juan J. Alonso's group at Stanford University.
#                 Prof. Piero Colonna's group at Delft University of Technology.
#                 Prof. Nicolas R. Gauger's group at Kaiserslautern University of Technology.
#                 Prof. Alberto Guardone's group at Polytechnic University of Milan.
#                 Prof. Rafael Palacios' group at Imperial College London.
#                 Prof. Edwin van der Weide's group at the University of Twente.
#                 Prof. Vincent Terrapon's group at the University of Liege.
#
# Copyright (C) 2012-2017 SU2, the open-source CFD code.
#
# SU2 is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# SU2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with SU2. If not, see <http://www.gnu.org/licenses/>.

import sys
from TestCase import TestCase

def main():
    '''This program runs SU2 and ensures that the output matches specified values.
       This will be used to do checks when code is pushed to github
       to make sure nothing is broken. '''

    test_list = []

    #####################################
    ### Disc. adj. compressible Euler ###
    #####################################

    # Inviscid NACA0012
    discadj_naca0012           = TestCase('discadj_naca0012')
    discadj_naca0012.cfg_dir   = "cont_adj_euler/naca0012"
    discadj_naca0012.cfg_file  = "inv_NACA0012_discadj.cfg"
    discadj_naca0012.test_iter = 100
    discadj_naca0012.test_vals = [-3.606841, -9.035214, -0.000000, 0.005688] #last 4 columns
    discadj_naca0012.su2_exec  = "parallel_computation.py -f"
    discadj_naca0012.timeout   = 1600
    discadj_naca0012.tol       = 0.00001
    test_list.append(discadj_naca0012)

    ####################################
    ### Disc. adj. compressible RANS ###
    ####################################

    # Adjoint turbulent NACA0012 SA
    discadj_rans_naca0012_sa           = TestCase('discadj_rans_naca0012_sa')
    discadj_rans_naca0012_sa.cfg_dir   = "disc_adj_rans/naca0012"
    discadj_rans_naca0012_sa.cfg_file  = "turb_NACA0012_sa.cfg"
    discadj_rans_naca0012_sa.test_iter = 10
    discadj_rans_naca0012_sa.test_vals = [-1.751959, 0.485790, 0.182890, -0.000018] #last 4 columns
    discadj_rans_naca0012_sa.su2_exec  = "parallel_computation.py -f"
    discadj_rans_naca0012_sa.timeout   = 1600
    discadj_rans_naca0012_sa.tol       = 0.00001
    test_list.append(discadj_rans_naca0012_sa)

    # Adjoint turbulent NACA0012 SST
    discadj_rans_naca0012_sst           = TestCase('discadj_rans_naca0012_sst')
    discadj_rans_naca0012_sst.cfg_dir   = "disc_adj_rans/naca0012"
    discadj_rans_naca0012_sst.cfg_file  = "turb_NACA0012_sst.cfg"
    discadj_rans_naca0012_sst.test_iter = 10
    discadj_rans_naca0012_sst.test_vals = [-1.655287, -0.497784, 0.129010, -0.000013] #last 4 columns
    discadj_rans_naca0012_sst.su2_exec  = "parallel_computation.py -f"
    discadj_rans_naca0012_sst.timeout   = 1600
    discadj_rans_naca0012_sst.tol       = 0.00001
    test_list.append(discadj_rans_naca0012_sst)

    #######################################################
    ### Unsteady Disc. adj. compressible RANS           ###
    #######################################################

    # Turbulent Cylinder
    discadj_cylinder           = TestCase('unsteady_cylinder')
    discadj_cylinder.cfg_dir   = "disc_adj_rans/cylinder"
    discadj_cylinder.cfg_file  = "cylinder.cfg"
    discadj_cylinder.test_iter = 10
    discadj_cylinder.test_vals = [3.527834, -1.781737, -0.012030, 0.000017] #last 4 columns
    discadj_cylinder.su2_exec  = "parallel_computation.py -f"
    discadj_cylinder.timeout   = 1600
    discadj_cylinder.tol       = 0.00001
    discadj_cylinder.unsteady  = True
    test_list.append(discadj_cylinder
)
    ######################################
    ### RUN TESTS                      ###
    ######################################

    pass_list = [ test.run_test() for test in test_list ]

    # Tests summary
    print '=================================================================='
    print 'Summary of the parallel tests'
    for i, test in enumerate(test_list):
        if (pass_list[i]):
            print '  passed - %s'%test.tag
        else:
            print '* FAILED - %s'%test.tag

    if all(pass_list):
        sys.exit(0)
    else:
        sys.exit(1)
    # done

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import csv
import itertools
from datetime import date, datetime, timedelta

from django.core.management.base import BaseCommand

from corehq.apps.enterprise.models import EnterprisePermissions
from corehq.apps.es import CaseES, FormES
from corehq.apps.es.aggregations import (
    DateHistogram,
    NestedAggregation,
    TermsAggregation,
)

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            'domains', nargs="*",
            help='Domains to check, will include enterprise-controlled child domains.'
        )
        parser.add_argument('--num-days', type=int, default=30, help='Number of days (UTC) to inspect')

    def handle(self, domains, **options):
        filename = "form_case_counts_{}".format(datetime.utcnow().strftime("%Y-%m-%d_%H.%M.%S"))
        case_types = sorted(_get_case_types(domains))
        with open(filename, 'w', encoding='utf-8') as csv_file:
            field_names = ['domain', 'date', 'form_submissions'] + case_types
            csv_writer = csv.DictWriter(csv_file, field_names)
            csv_writer.writeheader()
            for row in self.get_rows(domains, case_types, options['num_days']):
                csv_writer.writerow(row)
        print(f"Result saved to {filename}")

    def get_rows(self, domains, case_types, num_days):
        end = date.today()
        start = end - timedelta(days=num_days)
        for domain in _expand_domains(domains):
            submissions_counts = _get_submissions_counts(domain, start, end)
            case_update_counts = _get_case_update_counts(domain, start, end)

            day = start
            while day <= end:
                yield {
                    'domain': domain,
                    'date': day.isoformat(),
                    'form_submissions': submissions_counts.get(day, 0),
                    **{
                        case_type: case_update_counts.get((case_type, day), 0)
                        for case_type in case_types
                    }
                }
                day += timedelta(days=1)

def _expand_domains(domains):
    return sorted(set(itertools.chain(
        domains,
        *(EnterprisePermissions.get_domains(domain) for domain in domains)
    )))

def _get_datetime_range(num_days):
    now = datetime.utcnow()
    end = datetime(now.year, now.month, now.day)  # 00:00:00 this morning UTC
    start = end - timedelta(days=num_days)
    return start, end

def _get_case_types(domains):
    return (CaseES()
            .domain(domains)
            .terms_aggregation("type.exact", "case_types")
            .run().aggregations.case_types.keys)

def _get_submissions_counts(domain, start, end):
    res = (FormES()
           .domain(domain)
           .submitted(gte=start, lte=end)
           .aggregation(
               DateHistogram('date_histogram', 'received_on', DateHistogram.Interval.DAY))
           .run().aggregations.date_histogram)
    return {
        date.fromisoformat(bucket['key']): bucket['doc_count']
        for bucket in res.normalized_buckets
    }

def _get_case_update_counts(domain, start, end):
    res = (CaseES()
           .domain(domain)
           .active_in_range(gte=start, lte=end)
           .aggregation(
               TermsAggregation('case_types', 'type.exact')
               .aggregation(
                   NestedAggregation('actions', 'actions').aggregation(
                       DateHistogram('case_count', 'actions.server_date', DateHistogram.Interval.DAY)
                   )
               )
        )
           .run())

    ret = {}
    for case_type in res.aggregations.case_types.buckets_list:
        for case_count in case_type.actions.case_count.normalized_buckets:
            day = date.fromisoformat(case_count['key'])
            ret[(case_type.key, day)] = case_count['doc_count']
    return ret
<commit_msg>stickler<commit_after>import csv
import itertools
from datetime import date, datetime, timedelta

from django.core.management.base import BaseCommand

from corehq.apps.enterprise.models import EnterprisePermissions
from corehq.apps.es import CaseES, FormES
from corehq.apps.es.aggregations import (
    DateHistogram,
    NestedAggregation,
    TermsAggregation,
)

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            'domains', nargs="*",
            help='Domains to check, will include enterprise-controlled child domains.'
        )
        parser.add_argument('--num-days', type=int, default=30, help='Number of days (UTC) to inspect')

    def handle(self, domains, **options):
        filename = "form_case_counts_{}".format(datetime.utcnow().strftime("%Y-%m-%d_%H.%M.%S"))
        case_types = sorted(_get_case_types(domains))
        with open(filename, 'w', encoding='utf-8') as csv_file:
            field_names = ['domain', 'date', 'form_submissions'] + case_types
            csv_writer = csv.DictWriter(csv_file, field_names)
            csv_writer.writeheader()
            for row in self.get_rows(domains, case_types, options['num_days']):
                csv_writer.writerow(row)
        print(f"Result saved to {filename}")

    def get_rows(self, domains, case_types, num_days):
        end = date.today()
        start = end - timedelta(days=num_days)
        for domain in _expand_domains(domains):
            submissions_counts = _get_submissions_counts(domain, start, end)
            case_update_counts = _get_case_update_counts(domain, start, end)

            day = start
            while day <= end:
                yield {
                    'domain': domain,
                    'date': day.isoformat(),
                    'form_submissions': submissions_counts.get(day, 0),
                    **{
                        case_type: case_update_counts.get((case_type, day), 0)
                        for case_type in case_types
                    }
                }
                day += timedelta(days=1)

def _expand_domains(domains):
    return sorted(set(itertools.chain(
        domains,
        *(EnterprisePermissions.get_domains(domain) for domain in domains)
    )))

def _get_datetime_range(num_days):
    now = datetime.utcnow()
    end = datetime(now.year, now.month, now.day)  # 00:00:00 this morning UTC
    start = end - timedelta(days=num_days)
    return start, end

def _get_case_types(domains):
    return (CaseES()
            .domain(domains)
            .terms_aggregation("type.exact", "case_types")
            .run().aggregations.case_types.keys)

def _get_submissions_counts(domain, start, end):
    res = (FormES()
           .domain(domain)
           .submitted(gte=start, lte=end)
           .aggregation(
               DateHistogram('date_histogram', 'received_on', DateHistogram.Interval.DAY))
           .run().aggregations.date_histogram)
    return {
        date.fromisoformat(bucket['key']): bucket['doc_count']
        for bucket in res.normalized_buckets
    }

def _get_case_update_counts(domain, start, end):
    res = (CaseES()
           .domain(domain)
           .active_in_range(gte=start, lte=end)
           .aggregation(
               TermsAggregation('case_types', 'type.exact')
               .aggregation(
                   NestedAggregation('actions', 'actions').aggregation(
                       DateHistogram('case_count', 'actions.server_date', DateHistogram.Interval.DAY)
                   )
               )
           )
           .run())

    ret = {}
    for case_type in res.aggregations.case_types.buckets_list:
        for case_count in case_type.actions.case_count.normalized_buckets:
            day = date.fromisoformat(case_count['key'])
            ret[(case_type.key, day)] = case_count['doc_count']
    return ret
<|endoftext|>