<commit_before>#!/usr/bin/env python

# Amazon FPGA Hardware Development Kit
#
# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Amazon Software License (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
#    http://aws.amazon.com/asl/
#
# or in the "license" file accompanying this file. This file is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import base64
import boto3
import json
import os
import sys
import requests
import hmac

argparser = argparse.ArgumentParser(description="Notify via email or HTTP that FPGA CL build is complete")
argparser.add_argument('--email', help='Email to notify', default=os.environ.get('SNS_NOTIFY_EMAIL', None))
argparser.add_argument('--sns-notify-url', help='url to notify via SNS', default=os.environ.get('SNS_NOTIFY_URL', None))
argparser.add_argument('--notify-url', help='url to notify via POST', default=os.environ.get('NOTIFY_URL', None))
argparser.add_argument('--secret-key-file', help='File containing base64 encoded secret key for signing message sent to notify_url', default=os.environ.get('NOTIFY_SECRET_KEY_FILE', None))
argparser.add_argument('--build-user', help='User performing build', default=os.environ.get('BUILD_USER', 'default'))
argparser.add_argument('--build-project', help='Name of project built', default=os.environ.get('BUILD_PROJECT', None))
argparser.add_argument('--filename', help='Name of checkpoint archive', default=None)
argparser.add_argument('--timestamp', help='Timestamp of build', default=None)
argparser.add_argument('--sourcehash', help='md5sum of the RTL', default=None)
argparser.add_argument('--fpga-image-ids', help='JSON output from aws ec2 create-fpga-image', default=None)
options = argparser.parse_args()

sns = boto3.client('sns')
topic_resp = sns.create_topic(Name="FPGA_CL_BUILD_%s" % options.build_user)
print(topic_resp['TopicArn'])

if options.sns_notify_url:
    topic_resp_json = sns.create_topic(Name="FPGA_CL_BUILD_JSON")
    list_resp_json = sns.list_subscriptions_by_topic(TopicArn=topic_resp_json['TopicArn'])
    if not any(i['Endpoint'] == options.sns_notify_url for i in list_resp_json.get('Subscriptions')):
        print("Subscribing to the FPGA_CL_BUILD topic")
        sub_resp_json = sns.subscribe(TopicArn=topic_resp_json['TopicArn'], Protocol='http', Endpoint=options.sns_notify_url)
        print(sub_resp_json)

list_resp = sns.list_subscriptions_by_topic(TopicArn=topic_resp['TopicArn'])

if list_resp.get('Subscriptions'):
    print(list_resp.get('Subscriptions'))

if os.environ.get('EMAIL') is None:
    print('Please set your EMAIL environment variable to your email address.')
    sys.exit(1)

print("Using email address: %s" % os.environ.get('EMAIL'))

# subscribe if email is not in list
if not any(i['Endpoint'] == os.environ.get('EMAIL') for i in list_resp.get('Subscriptions')):
    print("Subscribing to the FPGA_CL_BUILD topic")
    sub_resp = sns.subscribe(TopicArn=topic_resp['TopicArn'], Protocol='email', Endpoint=os.environ.get('EMAIL'))
    print(sub_resp)

message_dict = { 'subject': 'Your FPGA CL build is complete.',
                 'user': options.build_user,
                 'email': options.email,
                 'project': options.build_project,
                 'filename': options.filename,
                 'timestamp': options.timestamp,
                 'sourceHash': options.sourcehash,
                 'fpgaImageId': '',
                 'fpgaImageGlobalId': '',
}
if options.fpga_image_ids:
    fpga_image_ids = json.loads(options.fpga_image_ids)
    message_dict['fpgaImageId'] = fpga_image_ids.get('FpgaImageId', '')
    message_dict['fpgaImageGlobalId'] = fpga_image_ids.get('FpgaImageGlobalId', '')

email_message_template = '''
Your FPGA CL build is complete:
    Project: %(project)s
    Timestamp: %(timestamp)s
    FPGA Image Id: %(fpgaImageId)s
    FPGA Image Global Id: %(fpgaImageGlobalId)s
'''

pub_resp = sns.publish(TopicArn=topic_resp['TopicArn'],
                       Message=email_message_template % message_dict,
                       Subject='Your FPGA CL build is complete.')
if options.sns_notify_url:
    print('notifying %s' % options.sns_notify_url);
    imageIds = json.loads(options.fpga_image_ids) if options.fpga_image_ids else None
    pub_resp = sns.publish(TopicArn=topic_resp_json['TopicArn'],
                           Message=json.dumps(message_dict),
                           Subject='Your FPGA CL build is complete.')

if options.notify_url:
    message = bytes(json.dumps(message_dict), 'utf8')
    signature = None
    if options.secret_key_file:
        secret_key_b64 = open(options.secret_key_file, 'r').read()
        secret_key = base64.b64decode(secret_key_b64)
        signature = hmac.new(secret_key, message).hexdigest()
    resp = requests.post(options.notify_url, data={'message': message, 'signature': signature })
    print('Posted to url %s got response %s' % (options.notify_url, resp))
    print(json.dumps({'message': message.decode('utf8'), 'signature': signature }))

sys.exit(0)
<commit_msg>using python3<commit_after>#!/usr/bin/env python3

# Amazon FPGA Hardware Development Kit
#
# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Amazon Software License (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
#    http://aws.amazon.com/asl/
#
# or in the "license" file accompanying this file. This file is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import base64
import boto3
import json
import os
import sys
import requests
import hmac

argparser = argparse.ArgumentParser(description="Notify via email or HTTP that FPGA CL build is complete")
argparser.add_argument('--email', help='Email to notify', default=os.environ.get('SNS_NOTIFY_EMAIL', None))
argparser.add_argument('--sns-notify-url', help='url to notify via SNS', default=os.environ.get('SNS_NOTIFY_URL', None))
argparser.add_argument('--notify-url', help='url to notify via POST', default=os.environ.get('NOTIFY_URL', None))
argparser.add_argument('--secret-key-file', help='File containing base64 encoded secret key for signing message sent to notify_url', default=os.environ.get('NOTIFY_SECRET_KEY_FILE', None))
argparser.add_argument('--build-user', help='User performing build', default=os.environ.get('BUILD_USER', 'default'))
argparser.add_argument('--build-project', help='Name of project built', default=os.environ.get('BUILD_PROJECT', None))
argparser.add_argument('--filename', help='Name of checkpoint archive', default=None)
argparser.add_argument('--timestamp', help='Timestamp of build', default=None)
argparser.add_argument('--sourcehash', help='md5sum of the RTL', default=None)
argparser.add_argument('--fpga-image-ids', help='JSON output from aws ec2 create-fpga-image', default=None)
options = argparser.parse_args()

sns = boto3.client('sns')
topic_resp = sns.create_topic(Name="FPGA_CL_BUILD_%s" % options.build_user)
print(topic_resp['TopicArn'])

if options.sns_notify_url:
    topic_resp_json = sns.create_topic(Name="FPGA_CL_BUILD_JSON")
    list_resp_json = sns.list_subscriptions_by_topic(TopicArn=topic_resp_json['TopicArn'])
    if not any(i['Endpoint'] == options.sns_notify_url for i in list_resp_json.get('Subscriptions')):
        print("Subscribing to the FPGA_CL_BUILD topic")
        sub_resp_json = sns.subscribe(TopicArn=topic_resp_json['TopicArn'], Protocol='http', Endpoint=options.sns_notify_url)
        print(sub_resp_json)

list_resp = sns.list_subscriptions_by_topic(TopicArn=topic_resp['TopicArn'])

if list_resp.get('Subscriptions'):
    print(list_resp.get('Subscriptions'))

if os.environ.get('EMAIL') is None:
    print('Please set your EMAIL environment variable to your email address.')
    sys.exit(1)

print("Using email address: %s" % os.environ.get('EMAIL'))

# subscribe if email is not in list
if not any(i['Endpoint'] == os.environ.get('EMAIL') for i in list_resp.get('Subscriptions')):
    print("Subscribing to the FPGA_CL_BUILD topic")
    sub_resp = sns.subscribe(TopicArn=topic_resp['TopicArn'], Protocol='email', Endpoint=os.environ.get('EMAIL'))
    print(sub_resp)

message_dict = { 'subject': 'Your FPGA CL build is complete.',
                 'user': options.build_user,
                 'email': options.email,
                 'project': options.build_project,
                 'filename': options.filename,
                 'timestamp': options.timestamp,
                 'sourceHash': options.sourcehash,
                 'fpgaImageId': '',
                 'fpgaImageGlobalId': '',
}
if options.fpga_image_ids:
    fpga_image_ids = json.loads(options.fpga_image_ids)
    message_dict['fpgaImageId'] = fpga_image_ids.get('FpgaImageId', '')
    message_dict['fpgaImageGlobalId'] = fpga_image_ids.get('FpgaImageGlobalId', '')

email_message_template = '''
Your FPGA CL build is complete:
    Project: %(project)s
    Timestamp: %(timestamp)s
    FPGA Image Id: %(fpgaImageId)s
    FPGA Image Global Id: %(fpgaImageGlobalId)s
'''

pub_resp = sns.publish(TopicArn=topic_resp['TopicArn'],
                       Message=email_message_template % message_dict,
                       Subject='Your FPGA CL build is complete.')
if options.sns_notify_url:
    print('notifying %s' % options.sns_notify_url);
    imageIds = json.loads(options.fpga_image_ids) if options.fpga_image_ids else None
    pub_resp = sns.publish(TopicArn=topic_resp_json['TopicArn'],
                           Message=json.dumps(message_dict),
                           Subject='Your FPGA CL build is complete.')

if options.notify_url:
    message = bytes(json.dumps(message_dict), 'utf8')
    signature = None
    if options.secret_key_file:
        secret_key_b64 = open(options.secret_key_file, 'r').read()
        secret_key = base64.b64decode(secret_key_b64)
        signature = hmac.new(secret_key, message).hexdigest()
    resp = requests.post(options.notify_url, data={'message': message, 'signature': signature })
    print('Posted to url %s got response %s' % (options.notify_url, resp))
    print(json.dumps({'message': message.decode('utf8'), 'signature': signature }))

sys.exit(0)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Evaluate model performance after training.
This is for comparison of supervised accuracy on different datasets.
Especially fro the plots for the broken data comparison.
"""

import numpy as np
from keras.models import load_model
import pickle
import os

from util.evaluation_utilities import *
from util.run_cnn import load_zero_center_data, h5_get_number_of_rows
from get_dataset_info import get_dataset_info

# "broken_data_mode_"+enc oder +unf
#e.g. "1enc"
which_one="2unf"
#instead of plotting acc vs. energy, one can also make a compare plot, which shows the difference
#between "on simulations" and "on measured data"
make_difference_plot=True

def get_info(which_one):
    if which_one=="1unf":
        #vgg_3_broken1_unf
        modelidents = ("vgg_3-broken1/trained_vgg_3-broken1_supervised_up_down_epoch6.h5",
                       "vgg_3-broken1/trained_vgg_3-broken1_supervised_up_down_epoch6.h5",
                       "vgg_3/trained_vgg_3_supervised_up_down_new_epoch5.h5")
        #Which dataset each to use
        dataset_array = ("xzt_broken", "xzt", "xzt")
        #Plot properties: All in the array are plotted in one figure, with own label each
        title_of_plot='Unfrozen network performance with manipulated simulations'
        #in the results/plots folder:
        plot_file_name = "vgg_3_broken1_unf.pdf"
        #y limits of plot:
        y_lims=(0.4,1.05)

    elif which_one=="1enc":
        #vgg_3_broken1_enc
        modelidents = ("vgg_3/trained_vgg_3_autoencoder_epoch10_supervised_up_down_broken1_epoch14.h5",
                       "vgg_3/trained_vgg_3_autoencoder_epoch10_supervised_up_down_broken1_epoch14.h5",
                       "vgg_3/trained_vgg_3_autoencoder_epoch10_supervised_up_down_accdeg_epoch24.h5")
        #Which dataset each to use
        dataset_array = ("xzt_broken", "xzt", "xzt")
        #Plot properties: All in the array are plotted in one figure, with own label each
        title_of_plot='Autoencoder-encoder network performance with manipulated simulations'
        #in the results/plots folder:
        plot_file_name = "vgg_3_broken1_enc.pdf"
        #y limits of plot:
        y_lims=(0.7,1.0)

    elif which_one=="2unf":
        #vgg_3_broken2_unf
        modelidents = ("vgg_3/trained_vgg_3_supervised_up_down_new_epoch5.h5",
                       "vgg_3/trained_vgg_3_supervised_up_down_new_epoch5.h5",
                       "vgg_3-noise10/trained_vgg_3-noise10_supervised_up_down_epoch6.h5")
        #Which dataset each to use
        dataset_array = ("xzt", "xzt_broken2", "xzt_broken2")
        #Plot properties: All in the array are plotted in one figure, with own label each
        title_of_plot='Unfrozen network performance with noisy data'
        #in the results/plots folder:
        plot_file_name = "vgg_3_broken2_unf.pdf"
        #y limits of plot:
        y_lims=(0.4,0.95)

    elif which_one=="2enc":
        #vgg_3_broken2_enc
        modelidents = ("vgg_3-noise10/trained_vgg_3-noise10_autoencoder_epoch10_supervised_up_down_epoch9.h5",
                       "vgg_3-noise10/trained_vgg_3-noise10_autoencoder_epoch10_supervised_up_down_epoch9.h5",
                       "vgg_3-noise10/trained_vgg_3-noise10_autoencoder_epoch10_supervised_up_down_noise_epoch14.h5")
        #Which dataset each to use
        dataset_array = ("xzt", "xzt_broken2", "xzt_broken2")
        #Plot properties: All in the array are plotted in one figure, with own label each
        title_of_plot='Autoencoder-encoder network performance with noisy data'
        #in the results/plots folder:
        plot_file_name = "vgg_3_broken2_enc.pdf"
        #y limits of plot:
        y_lims=(0.4,0.95)

    else:
        print(which_one, "is not known!")
        raise(TypeError)

    return modelidents,dataset_array,title_of_plot,plot_file_name,y_lims

modelidents,dataset_array,title_of_plot,plot_file_name,y_lims = get_info(which_one)

label_array=["On 'simulations'", "On 'measured' data", "Upper limit on 'measured' data"]
#Overwrite default color palette. Leave empty for auto
color_array=["orange", "blue", "navy"]
#loss, acc, None
plot_type = "acc"
#Info about model
class_type = (2, 'up_down')

#XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

modelpath = "/home/woody/capn/mppi013h/Km3-Autoencoder/models/"
plot_path = "/home/woody/capn/mppi013h/Km3-Autoencoder/results/plots/"

modelnames=[] # a tuple of eg       "vgg_1_xzt_supervised_up_down_epoch6"
#           (created from   "trained_vgg_1_xzt_supervised_up_down_epoch6.h5"   )
for modelident in modelidents:
    modelnames.append(modelident.split("trained_")[1][:-3])

save_plot_as = plot_path + plot_file_name

#Accuracy as a function of energy binned to a histogramm. It is dumped automatically into the
#results/data folder, so that it has not to be generated again
def make_and_save_hist_data(modelpath, dataset, modelident, class_type, name_of_file):
    model = load_model(modelpath + modelident)

    dataset_info_dict = get_dataset_info(dataset)
    #home_path=dataset_info_dict["home_path"]
    train_file=dataset_info_dict["train_file"]
    test_file=dataset_info_dict["test_file"]
    n_bins=dataset_info_dict["n_bins"]
    broken_simulations_mode=dataset_info_dict["broken_simulations_mode"]

    train_tuple=[[train_file, h5_get_number_of_rows(train_file)]]
    xs_mean = load_zero_center_data(train_files=train_tuple, batchsize=32, n_bins=n_bins, n_gpu=1)

    print("Making energy_coorect_array of ", modelident)
    arr_energy_correct = make_performance_array_energy_correct(model=model, f=test_file, n_bins=n_bins, class_type=class_type, xs_mean=xs_mean, batchsize = 32, broken_simulations_mode=broken_simulations_mode, swap_4d_channels=None, samples=None, dataset_info_dict=dataset_info_dict)
    #hist_data = [bin_edges_centered, hist_1d_energy_accuracy_bins]:
    hist_data = make_energy_to_accuracy_data(arr_energy_correct, plot_range=(3,100))
    #save to file
    print("Saving hist_data as", name_of_file)
    with open(name_of_file, "wb") as dump_file:
        pickle.dump(hist_data, dump_file)
    return hist_data

"""
#Loss of an AE as a function of energy, rest like above
def make_and_save_hist_data_autoencoder(modelpath, modelident, modelname, test_file, n_bins, class_type, xs_mean):
    model = load_model(modelpath + modelident)
    print("Making and saving energy_loss_array of autoencoder ", modelname)
    hist_data = make_autoencoder_energy_data(model=model, f=test_file, n_bins=n_bins, class_type=class_type, xs_mean=xs_mean, batchsize = 32, swap_4d_channels=None, samples=None)
    #save to file
    with open("/home/woody/capn/mppi013h/Km3-Autoencoder/results/data/" + modelname + "_hist_data.txt", "wb") as dump_file:
        pickle.dump(hist_data, dump_file)
    return hist_data
"""

#open dumped histogramm data, that was generated from the above two functions
def open_hist_data(name_of_file):
    #hist data is list len 2 with 0:energy array, 1:acc/loss array
    print("Opening existing hist_data file", name_of_file)
    #load again
    with open(name_of_file, "rb") as dump_file:
        hist_data = pickle.load(dump_file)
    return hist_data

def make_or_load_files(modelnames, dataset_array, modelidents=None, modelpath=None, class_type=None):
    hist_data_array=[]
    for i,modelname in enumerate(modelnames):
        dataset=dataset_array[i]

        print("Working on ",modelname,"using dataset", dataset)
        name_of_file="/home/woody/capn/mppi013h/Km3-Autoencoder/results/data/" + modelname + "_" + dataset + "_hist_data.txt"
        if os.path.isfile(name_of_file)==True:
            hist_data_array.append(open_hist_data(name_of_file))
        else:
            hist_data = make_and_save_hist_data(modelpath, dataset, modelidents[i], class_type, name_of_file)
            hist_data_array.append(hist_data)
        print("Done.")
    return hist_data_array

if make_difference_plot == False:
    #generate or load data automatically:
    hist_data_array = make_or_load_files(modelnames, dataset_array, modelidents, modelpath, class_type)
    #make plot of multiple data:
    if plot_type == "acc":
        y_label_of_plot="Accuracy"
        make_energy_to_accuracy_plot_comp_data(hist_data_array, label_array, title_of_plot, filepath=save_plot_as, y_label=y_label_of_plot, y_lims=y_lims, color_array=color_array)
    elif plot_type == "loss":
        y_label_of_plot="Loss"
        make_energy_to_loss_plot_comp_data(hist_data_array, label_array, title_of_plot, filepath=save_plot_as, y_label=y_label_of_plot, color_array=color_array)
    elif plot_type == None:
        print("plot_type==None: Not generating plots")
    else:
        print("Plot type", plot_type, "not supported. Not generating plots, but hist_data is still saved.")

    print("Plot saved to", save_plot_as)
else:
    #which plots to make diff of; first - second
    make_diff_of_list=((0,1),)

    #label_array=["On 'simulations'", "On 'measured' data", "Upper limit on 'measured' data"]
    modelidents,dataset_array,title_of_plot,plot_file_name,y_lims = get_info("2unf")

    modelnames=[] # a tuple of eg       "vgg_1_xzt_supervised_up_down_epoch6"
    #           (created from   "trained_vgg_1_xzt_supervised_up_down_epoch6.h5"   )
    for modelident in modelidents:
        modelnames.append(modelident.split("trained_")[1][:-3])

    hist_data_array_unf = make_or_load_files(modelnames, dataset_array, modelidents, modelpath, class_type)

    modelidents,dataset_array,title_of_plot,plot_file_name,y_lims = get_info("2enc")

    modelnames=[] # a tuple of eg       "vgg_1_xzt_supervised_up_down_epoch6"
    #           (created from   "trained_vgg_1_xzt_supervised_up_down_epoch6.h5"   )
    for modelident in modelidents:
        modelnames.append(modelident.split("trained_")[1][:-3])

    hist_data_array_enc = make_or_load_files(modelnames, dataset_array, modelidents, modelpath, class_type)

    label_array=["Unfrozen", "Autoencoder-encoder"]
    #Overwrite default color palette. Leave empty for auto
    color_array=[]
    #loss, acc, None
    plot_type = "acc"
    #Info about model
    class_type = (2, 'up_down')

    modelpath = "/home/woody/capn/mppi013h/Km3-Autoencoder/models/"
    plot_path = "/home/woody/capn/mppi013h/Km3-Autoencoder/results/plots/"

    title_of_plot="Loss of accuracy when moving from 'simulations' to 'measured' data"
    save_plot_as = plot_path + "vgg_3_broken2_sim_real.pdf"
    y_lims=(-0.04,0.09)

    hist_data_array_diff=[]
    for make_diff_of in make_diff_of_list:
        hist_1=hist_data_array_unf[make_diff_of[0]]
        hist_2=hist_data_array_unf[make_diff_of[1]]
        diff_hist=[hist_1[0], hist_1[1]-hist_2[1]]
        hist_data_array_diff.append(diff_hist)

        hist_1=hist_data_array_enc[make_diff_of[0]]
        hist_2=hist_data_array_enc[make_diff_of[1]]
        diff_hist=[hist_1[0], hist_1[1]-hist_2[1]]
        hist_data_array_diff.append(diff_hist)

    #make plot of multiple data:
    if plot_type == "acc":
        y_label_of_plot="Difference in accuracy"
        make_energy_to_accuracy_plot_comp_data(hist_data_array_diff, label_array, title_of_plot, filepath=save_plot_as, y_label=y_label_of_plot, y_lims=y_lims, color_array=color_array)
    elif plot_type == "loss":
        y_label_of_plot="Loss"
        make_energy_to_loss_plot_comp_data(hist_data_array_diff, label_array, title_of_plot, filepath=save_plot_as, y_label=y_label_of_plot, color_array=color_array)
    elif plot_type == None:
        print("plot_type==None: Not generating plots")
    else:
        print("Plot type", plot_type, "not supported. Not generating plots, but hist_data is still saved.")

    print("Plot saved to", save_plot_as)
<commit_msg>small change<commit_after># -*- coding: utf-8 -*-

"""
Evaluate model performance after training.
This is for comparison of supervised accuracy on different datasets.
Especially fro the plots for the broken data comparison.
"""

import numpy as np
from keras.models import load_model
import pickle
import os

from util.evaluation_utilities import *
from util.run_cnn import load_zero_center_data, h5_get_number_of_rows
from get_dataset_info import get_dataset_info

# "broken_data_mode_"+enc oder +unf
#e.g. "1enc"
which_one="2unf"
#instead of plotting acc vs. energy, one can also make a compare plot, which shows the difference
#between "on simulations" and "on measured data"
make_difference_plot=True

def get_info(which_one):
    if which_one=="1unf":
        #vgg_3_broken1_unf
        modelidents = ("vgg_3-broken1/trained_vgg_3-broken1_supervised_up_down_epoch6.h5",
                       "vgg_3-broken1/trained_vgg_3-broken1_supervised_up_down_epoch6.h5",
                       "vgg_3/trained_vgg_3_supervised_up_down_new_epoch5.h5")
        #Which dataset each to use
        dataset_array = ("xzt_broken", "xzt", "xzt")
        #Plot properties: All in the array are plotted in one figure, with own label each
        title_of_plot='Unfrozen network performance with manipulated simulations'
        #in the results/plots folder:
        plot_file_name = "vgg_3_broken1_unf.pdf"
        #y limits of plot:
        y_lims=(0.4,1.05)

    elif which_one=="1enc":
        #vgg_3_broken1_enc
        modelidents = ("vgg_3/trained_vgg_3_autoencoder_epoch10_supervised_up_down_broken1_epoch14.h5",
                       "vgg_3/trained_vgg_3_autoencoder_epoch10_supervised_up_down_broken1_epoch14.h5",
                       "vgg_3/trained_vgg_3_autoencoder_epoch10_supervised_up_down_accdeg_epoch24.h5")
        #Which dataset each to use
        dataset_array = ("xzt_broken", "xzt", "xzt")
        #Plot properties: All in the array are plotted in one figure, with own label each
        title_of_plot='Autoencoder-encoder network performance with manipulated simulations'
        #in the results/plots folder:
        plot_file_name = "vgg_3_broken1_enc.pdf"
        #y limits of plot:
        y_lims=(0.7,1.0)

    elif which_one=="2unf":
        #vgg_3_broken2_unf
        modelidents = ("vgg_3/trained_vgg_3_supervised_up_down_new_epoch5.h5",
                       "vgg_3/trained_vgg_3_supervised_up_down_new_epoch5.h5",
                       "vgg_3-noise10/trained_vgg_3-noise10_supervised_up_down_epoch6.h5")
        #Which dataset each to use
        dataset_array = ("xzt", "xzt_broken2", "xzt_broken2")
        #Plot properties: All in the array are plotted in one figure, with own label each
        title_of_plot='Unfrozen network performance with noisy data'
        #in the results/plots folder:
        plot_file_name = "vgg_3_broken2_unf.pdf"
        #y limits of plot:
        y_lims=(0.4,0.95)

    elif which_one=="2enc":
        #vgg_3_broken2_enc
        modelidents = ("vgg_3-noise10/trained_vgg_3-noise10_autoencoder_epoch10_supervised_up_down_epoch9.h5",
                       "vgg_3-noise10/trained_vgg_3-noise10_autoencoder_epoch10_supervised_up_down_epoch9.h5",
                       "vgg_3-noise10/trained_vgg_3-noise10_autoencoder_epoch10_supervised_up_down_noise_epoch14.h5")
        #Which dataset each to use
        dataset_array = ("xzt", "xzt_broken2", "xzt_broken2")
        #Plot properties: All in the array are plotted in one figure, with own label each
        title_of_plot='Autoencoder-encoder network performance with noisy data'
        #in the results/plots folder:
        plot_file_name = "vgg_3_broken2_enc.pdf"
        #y limits of plot:
        y_lims=(0.4,0.95)

    else:
        print(which_one, "is not known!")
        raise(TypeError)

    return modelidents,dataset_array,title_of_plot,plot_file_name,y_lims

modelidents,dataset_array,title_of_plot,plot_file_name,y_lims = get_info(which_one)

label_array=["On 'simulations'", "On 'measured' data", "Upper limit on 'measured' data"]
#Overwrite default color palette. Leave empty for auto
color_array=["orange", "blue", "navy"]
#loss, acc, None
plot_type = "acc"
#Info about model
class_type = (2, 'up_down')

#XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

modelpath = "/home/woody/capn/mppi013h/Km3-Autoencoder/models/"
plot_path = "/home/woody/capn/mppi013h/Km3-Autoencoder/results/plots/"

modelnames=[] # a tuple of eg       "vgg_1_xzt_supervised_up_down_epoch6"
#           (created from   "trained_vgg_1_xzt_supervised_up_down_epoch6.h5"   )
for modelident in modelidents:
    modelnames.append(modelident.split("trained_")[1][:-3])

save_plot_as = plot_path + plot_file_name

#Accuracy as a function of energy binned to a histogramm. It is dumped automatically into the
#results/data folder, so that it has not to be generated again
def make_and_save_hist_data(modelpath, dataset, modelident, class_type, name_of_file):
    model = load_model(modelpath + modelident)

    dataset_info_dict = get_dataset_info(dataset)
    #home_path=dataset_info_dict["home_path"]
    train_file=dataset_info_dict["train_file"]
    test_file=dataset_info_dict["test_file"]
    n_bins=dataset_info_dict["n_bins"]
    broken_simulations_mode=dataset_info_dict["broken_simulations_mode"]

    train_tuple=[[train_file, h5_get_number_of_rows(train_file)]]
    xs_mean = load_zero_center_data(train_files=train_tuple, batchsize=32, n_bins=n_bins, n_gpu=1)

    print("Making energy_coorect_array of ", modelident)
    arr_energy_correct = make_performance_array_energy_correct(model=model, f=test_file, n_bins=n_bins, class_type=class_type, xs_mean=xs_mean, batchsize = 32, broken_simulations_mode=broken_simulations_mode, swap_4d_channels=None, samples=None, dataset_info_dict=dataset_info_dict)
    #hist_data = [bin_edges_centered, hist_1d_energy_accuracy_bins]:
    hist_data = make_energy_to_accuracy_data(arr_energy_correct, plot_range=(3,100))
    #save to file
    print("Saving hist_data as", name_of_file)
    with open(name_of_file, "wb") as dump_file:
        pickle.dump(hist_data, dump_file)
    return hist_data

"""
#Loss of an AE as a function of energy, rest like above
def make_and_save_hist_data_autoencoder(modelpath, modelident, modelname, test_file, n_bins, class_type, xs_mean):
    model = load_model(modelpath + modelident)
    print("Making and saving energy_loss_array of autoencoder ", modelname)
    hist_data = make_autoencoder_energy_data(model=model, f=test_file, n_bins=n_bins, class_type=class_type, xs_mean=xs_mean, batchsize = 32, swap_4d_channels=None, samples=None)
    #save to file
    with open("/home/woody/capn/mppi013h/Km3-Autoencoder/results/data/" + modelname + "_hist_data.txt", "wb") as dump_file:
        pickle.dump(hist_data, dump_file)
    return hist_data
"""

#open dumped histogramm data, that was generated from the above two functions
def open_hist_data(name_of_file):
    #hist data is list len 2 with 0:energy array, 1:acc/loss array
    print("Opening existing hist_data file", name_of_file)
    #load again
    with open(name_of_file, "rb") as dump_file:
        hist_data = pickle.load(dump_file)
    return hist_data

def make_or_load_files(modelnames, dataset_array, modelidents=None, modelpath=None, class_type=None):
    hist_data_array=[]
    for i,modelname in enumerate(modelnames):
        dataset=dataset_array[i]

        print("Working on ",modelname,"using dataset", dataset)
        name_of_file="/home/woody/capn/mppi013h/Km3-Autoencoder/results/data/" + modelname + "_" + dataset + "_hist_data.txt"
        if os.path.isfile(name_of_file)==True:
            hist_data_array.append(open_hist_data(name_of_file))
        else:
            hist_data = make_and_save_hist_data(modelpath, dataset, modelidents[i], class_type, name_of_file)
            hist_data_array.append(hist_data)
        print("Done.")
    return hist_data_array

if make_difference_plot == False:
    #generate or load data automatically:
    hist_data_array = make_or_load_files(modelnames, dataset_array, modelidents, modelpath, class_type)
    #make plot of multiple data:
    if plot_type == "acc":
        y_label_of_plot="Accuracy"
        make_energy_to_accuracy_plot_comp_data(hist_data_array, label_array, title_of_plot, filepath=save_plot_as, y_label=y_label_of_plot, y_lims=y_lims, color_array=color_array)
    elif plot_type == "loss":
        y_label_of_plot="Loss"
        make_energy_to_loss_plot_comp_data(hist_data_array, label_array, title_of_plot, filepath=save_plot_as, y_label=y_label_of_plot, color_array=color_array)
    elif plot_type == None:
        print("plot_type==None: Not generating plots")
    else:
        print("Plot type", plot_type, "not supported. Not generating plots, but hist_data is still saved.")

    print("Plot saved to", save_plot_as)
else:
    #which plots to make diff of; first - second
    make_diff_of_list=((0,1),(2,1))
    title_list=("Loss of accuracy when moving from 'simulations' to 'measured' data",
                "DIfference in accuracy: Upper limit - 'measured' data")
    save_as_list=(plot_path + "vgg_3_broken2_sim_real.pdf",
                  plot_path + "vgg_3_broken2_upper_real.pdf")
    y_lims_list=((-0.04,0.09),(-0.03,0.06))

    for i in range(len(make_diff_of_list)):
        #label_array=["On 'simulations'", "On 'measured' data", "Upper limit on 'measured' data"]
        modelidents,dataset_array,title_of_plot,plot_file_name,y_lims = get_info("2unf")

        modelnames=[] # a tuple of eg       "vgg_1_xzt_supervised_up_down_epoch6"
        #           (created from   "trained_vgg_1_xzt_supervised_up_down_epoch6.h5"   )
        for modelident in modelidents:
            modelnames.append(modelident.split("trained_")[1][:-3])

        hist_data_array_unf = make_or_load_files(modelnames, dataset_array, modelidents, modelpath, class_type)

        modelidents,dataset_array,title_of_plot,plot_file_name,y_lims = get_info("2enc")

        modelnames=[] # a tuple of eg       "vgg_1_xzt_supervised_up_down_epoch6"
        #           (created from   "trained_vgg_1_xzt_supervised_up_down_epoch6.h5"   )
        for modelident in modelidents:
            modelnames.append(modelident.split("trained_")[1][:-3])

        hist_data_array_enc = make_or_load_files(modelnames, dataset_array, modelidents, modelpath, class_type)

        label_array=["Unfrozen", "Autoencoder-encoder"]
        #Overwrite default color palette. Leave empty for auto
        color_array=[]
        #loss, acc, None
        plot_type = "acc"
        #Info about model
        class_type = (2, 'up_down')

        modelpath = "/home/woody/capn/mppi013h/Km3-Autoencoder/models/"
        plot_path = "/home/woody/capn/mppi013h/Km3-Autoencoder/results/plots/"

        title_of_plot=title_list[i]
        save_plot_as = save_as_list[i]
        y_lims=y_lims_list[i]
        make_diff_of=make_diff_of_list[i]

        hist_data_array_diff=[]
        hist_1=hist_data_array_unf[make_diff_of[0]]
        hist_2=hist_data_array_unf[make_diff_of[1]]
        diff_hist=[hist_1[0], hist_1[1]-hist_2[1]]
        hist_data_array_diff.append(diff_hist)

        hist_1=hist_data_array_enc[make_diff_of[0]]
        hist_2=hist_data_array_enc[make_diff_of[1]]
        diff_hist=[hist_1[0], hist_1[1]-hist_2[1]]
        hist_data_array_diff.append(diff_hist)

        #make plot of multiple data:
        if plot_type == "acc":
            y_label_of_plot="Difference in accuracy"
            make_energy_to_accuracy_plot_comp_data(hist_data_array_diff, label_array, title_of_plot, filepath=save_plot_as, y_label=y_label_of_plot, y_lims=y_lims, color_array=color_array)
        elif plot_type == "loss":
            y_label_of_plot="Loss"
            make_energy_to_loss_plot_comp_data(hist_data_array_diff, label_array, title_of_plot, filepath=save_plot_as, y_label=y_label_of_plot, color_array=color_array)
        elif plot_type == None:
            print("plot_type==None: Not generating plots")
        else:
            print("Plot type", plot_type, "not supported. Not generating plots, but hist_data is still saved.")

        print("Plot saved to", save_plot_as)
<|endoftext|>


<commit_before># Create a script to run a random hyperparameter search.

import copy
import getpass
import os
import random
import numpy as np
import gflags
import sys

NYU_NON_PBS = False
NAME = "listops_06_12_64p"
SWEEP_RUNS = 8

LIN = "LIN"
EXP = "EXP"
BOOL = "BOOL"
CHOICE = "CHOICE"
SS_BASE = "SS_BASE"

FLAGS = gflags.FLAGS

gflags.DEFINE_string("training_data_path", "spinn/data/listops/train_d20s.tsv", "")
gflags.DEFINE_string("eval_data_path", "spinn/data/listops/test_d20s.tsv", "")
gflags.DEFINE_string("log_path", "/home/sb6065/logs", "")

FLAGS(sys.argv)

# Instructions: Configure the variables in this block, then run
# the following on a machine with qsub access:
# python make_sweep.py > my_sweep.sh
# bash my_sweep.sh

# - #

# Non-tunable flags that must be passed in.

FIXED_PARAMETERS = {
    "data_type":     "listops",
    "model_type":      "Pyramid",
    "training_data_path":    FLAGS.training_data_path,
    "eval_data_path":    FLAGS.eval_data_path,
    "log_path": FLAGS.log_path,
    "metrics_path": FLAGS.log_path,
    "ckpt_path":  FLAGS.log_path,
    "word_embedding_dim":   "128",
    "model_dim":   "128",
    "seq_length":   "50",
    "eval_seq_length":  "80",
    "eval_interval_steps": "100",
    "statistics_interval_steps": "100",
    #"use_internal_parser": "",
    "batch_size":  "64",
    #"nouse_tracking_in_composition": "",
    #"nolateral_tracking": "",
    "encode": "pass",
    "mlp_dim": "16",
    "num_mlp_layers": "2",
    #"transition_weight": "0.1",
    "semantic_classifier_keep_rate": "1.0",
    "embedding_keep_rate": "1.0",
}

# Tunable parameters.
SWEEP_PARAMETERS = {
    "learning_rate":      ("lr", EXP, 0.0001, 0.001),  # RNN likes higher, but below 009.
    "l2_lambda":          ("l2", EXP, 8e-7, 2e-5),
    "learning_rate_decay_per_10k_steps": ("dec", EXP, 0.4, 0.9),
    "pyramid_selection_keep_rate": ("pykr", LIN, 0.8, 1.0),
    "pyramid_test_time_temperature_multiplier": ("tttm", EXP, 0.0001, 1.0),
    "pyramid_trainable_temperature": ("tt", BOOL, None, None),
#    "tracking_lstm_hidden_dim": ("tdim", EXP, 4, 16),
#    "rl_weight":  ("rlwt", EXP, 0.000001, 0.0009),
#    "transition_weight":  ("trwt", EXP, 0.3, 3.0),
}

sweep_name = "sweep_" + NAME + "_" + \
    FIXED_PARAMETERS["data_type"] + "_" + FIXED_PARAMETERS["model_type"]

# - #
print "# NAME: " + sweep_name
print "# NUM RUNS: " + str(SWEEP_RUNS)
print "# SWEEP PARAMETERS: " + str(SWEEP_PARAMETERS)
print "# FIXED_PARAMETERS: " + str(FIXED_PARAMETERS)
print

for run_id in range(SWEEP_RUNS):
    params = {}
    name = sweep_name + "_" + str(run_id)

    params.update(FIXED_PARAMETERS)
    # Any param appearing in both sets will be overwritten by the sweep value.

    for param in SWEEP_PARAMETERS:
        config = SWEEP_PARAMETERS[param]
        t = config[1]
        mn = config[2]
        mx = config[3]

        r = random.uniform(0, 1)
        if t == EXP:
            lmn = np.log(mn)
            lmx = np.log(mx)
            sample = np.exp(lmn + (lmx - lmn) * r)
        elif t == BOOL:
            sample = r > 0.5
        elif t==SS_BASE:
            lmn = np.log(mn)
            lmx = np.log(mx)
            sample = 1 - np.exp(lmn + (lmx - lmn) * r)
        elif t==CHOICE:
            sample = random.choice(mn)
        else:
            sample = mn + (mx - mn) * r

        if isinstance(mn, int):
            sample = int(round(sample, 0))
            val_disp = str(sample)
            params[param] = sample
        elif isinstance(mn, float):
            val_disp = "%.2g" % sample
            params[param] = sample
        elif t==BOOL:
            val_disp = str(int(sample))
            if not sample:
                params['no' + param] = ''
            else:
                params[param] = ''
        else:
            val_disp = sample
            params[param] = sample
        name += "-" + config[0] + val_disp

    flags = ""
    for param in params:
        value = params[param]
        flags += " --" + param + " " + str(value)

    flags += " --experiment_name " + name
    if NYU_NON_PBS:
        print "cd spinn/python; python2.7 -m spinn.models.supervised_classifier " + flags
    else:
        print "SPINN_FLAGS=\"" + flags + "\" bash ../scripts/sbatch_submit.sh"
    print
<commit_msg>Sweep update.<commit_after># Create a script to run a random hyperparameter search.

import copy
import getpass
import os
import random
import numpy as np
import gflags
import sys

NYU_NON_PBS = False
NAME = "listops_06_15_64p"
SWEEP_RUNS = 8

LIN = "LIN"
EXP = "EXP"
BOOL = "BOOL"
CHOICE = "CHOICE"
SS_BASE = "SS_BASE"

FLAGS = gflags.FLAGS

gflags.DEFINE_string("training_data_path", "spinn/data/listops/train_d20s.tsv", "")
gflags.DEFINE_string("eval_data_path", "spinn/data/listops/test_d20s.tsv", "")
gflags.DEFINE_string("log_path", "/home/sb6065/logs", "")

FLAGS(sys.argv)

# Instructions: Configure the variables in this block, then run
# the following on a machine with qsub access:
# python make_sweep.py > my_sweep.sh
# bash my_sweep.sh

# - #

# Non-tunable flags that must be passed in.

FIXED_PARAMETERS = {
    "data_type":     "listops",
    "model_type":      "Pyramid",
    "training_data_path":    FLAGS.training_data_path,
    "eval_data_path":    FLAGS.eval_data_path,
    "log_path": FLAGS.log_path,
    "metrics_path": FLAGS.log_path,
    "ckpt_path":  FLAGS.log_path,
    "word_embedding_dim":   "128",
    "model_dim":   "128",
    "seq_length":   "50",
    "eval_seq_length":  "80",
    "eval_interval_steps": "100",
    "statistics_interval_steps": "100",
    #"use_internal_parser": "",
    "batch_size":  "64",
    #"nouse_tracking_in_composition": "",
    #"nolateral_tracking": "",
    "encode": "pass",
    "mlp_dim": "16",
    "num_mlp_layers": "2",
    #"transition_weight": "0.1",
    "semantic_classifier_keep_rate": "1.0",
    "embedding_keep_rate": "1.0",
    "pyramid_selection_keep_rate": "1.0",
}

# Tunable parameters.
SWEEP_PARAMETERS = {
    "learning_rate":      ("lr", EXP, 0.0001, 0.001),  # RNN likes higher, but below 009.
    "l2_lambda":          ("l2", EXP, 8e-7, 2e-5),
    "learning_rate_decay_per_10k_steps": ("dec", EXP, 0.4, 0.8),
    "pyramid_test_time_temperature_multiplier": ("tttm", EXP, 0.0001, 0.1),
    "pyramid_trainable_temperature": ("tt", BOOL, None, None),
#    "tracking_lstm_hidden_dim": ("tdim", EXP, 4, 16),
#    "rl_weight":  ("rlwt", EXP, 0.000001, 0.0009),
#    "transition_weight":  ("trwt", EXP, 0.3, 3.0),
}

sweep_name = "sweep_" + NAME + "_" + \
    FIXED_PARAMETERS["data_type"] + "_" + FIXED_PARAMETERS["model_type"]

# - #
print "# NAME: " + sweep_name
print "# NUM RUNS: " + str(SWEEP_RUNS)
print "# SWEEP PARAMETERS: " + str(SWEEP_PARAMETERS)
print "# FIXED_PARAMETERS: " + str(FIXED_PARAMETERS)
print

for run_id in range(SWEEP_RUNS):
    params = {}
    name = sweep_name + "_" + str(run_id)

    params.update(FIXED_PARAMETERS)
    # Any param appearing in both sets will be overwritten by the sweep value.

    for param in SWEEP_PARAMETERS:
        config = SWEEP_PARAMETERS[param]
        t = config[1]
        mn = config[2]
        mx = config[3]

        r = random.uniform(0, 1)
        if t == EXP:
            lmn = np.log(mn)
            lmx = np.log(mx)
            sample = np.exp(lmn + (lmx - lmn) * r)
        elif t == BOOL:
            sample = r > 0.5
        elif t==SS_BASE:
            lmn = np.log(mn)
            lmx = np.log(mx)
            sample = 1 - np.exp(lmn + (lmx - lmn) * r)
        elif t==CHOICE:
            sample = random.choice(mn)
        else:
            sample = mn + (mx - mn) * r

        if isinstance(mn, int):
            sample = int(round(sample, 0))
            val_disp = str(sample)
            params[param] = sample
        elif isinstance(mn, float):
            val_disp = "%.2g" % sample
            params[param] = sample
        elif t==BOOL:
            val_disp = str(int(sample))
            if not sample:
                params['no' + param] = ''
            else:
                params[param] = ''
        else:
            val_disp = sample
            params[param] = sample
        name += "-" + config[0] + val_disp

    flags = ""
    for param in params:
        value = params[param]
        flags += " --" + param + " " + str(value)

    flags += " --experiment_name " + name
    if NYU_NON_PBS:
        print "cd spinn/python; python2.7 -m spinn.models.supervised_classifier " + flags
    else:
        print "SPINN_FLAGS=\"" + flags + "\" bash ../scripts/sbatch_submit.sh"
    print
<|endoftext|>


<commit_before># coding: utf-8

""" Observe particles from a given simulation """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys
import copy
import logging
import random

# Third-party
import astropy.units as u
from astropy.utils.console import color_print
import emcee
import h5py
import matplotlib.pyplot as plt
import numpy as np
np.seterr(all="ignore")
import scipy
scipy.seterr(all="ignore")
import yaml

# Project
from streams import usys
from streams.coordinates.frame import heliocentric, galactocentric
from streams.dynamics import ObservedParticle
import streams.io as s_io
from streams.observation.gaia import gaia_spitzer_errors
from streams.potential.lm10 import LawMajewski2010
from streams.util import _parse_quantity
from streams.integrate import LeapfrogIntegrator

# Create logger
logger = logging.getLogger(__name__)

def observe_simulation(class_name, particle_error_model=None, satellite_error_model=None,
                       selection_expr=None, N=None, output_file=None, overwrite=False,
                       seed=None, class_kwargs=dict()):
    """ Observe simulation data and write the output to a standard HDF5 format.

        TODO: handle missing dimensions

        Parameters
        ----------

    """

    if os.path.exists(output_file) and overwrite:
        os.remove(output_file)

    if os.path.exists(output_file):
        raise IOError("File '{}' already exists! Did you "
                      "want to use overwrite=True?".format(output_file))

    # read the simulation data from the specified class
    if seed is None:
        seed = np.random.randint(100)

    logger.debug("Using seed: {}".format(seed))
    np.random.seed(seed)
    random.seed(seed)

    try:
        Simulation = getattr(s_io, class_name)
    except AttributeError:
        raise ValueError("Simulation class '{}' not found!".format(class_name))

    # instantiate the specified simulation class with any additional keyword
    #   arguments, e.g., mass from SgrSimulation
    simulation = Simulation(**class_kwargs)

    # read particles from the simulation class
    sim_time = simulation.particle_units[0]/simulation.particle_units[-1]

    # HACK HACK HACK
    selection_expr = "(tub!=0) & (tub>{}) & (tub<{}) & (sqrt((x+8)**2 + y**2 + z**2)<75)"\
                     .format((1750*u.Myr).to(sim_time).value,
                             (4700*u.Myr).to(sim_time).value)
    particles = simulation.particles(N=N, expr=selection_expr)

    logger.debug("Read in {} particles with expr='{}'"\
                 .format(particles.nparticles, selection_expr))

    # read the satellite position
    satellite = simulation.satellite()
    logger.debug("Read in present position of satellite...")

    ################################################################
    # HACK: Figure out if in leading or trailing tail
    s = satellite
    p = particles
    lm10_potential = LawMajewski2010()
    X = np.vstack((s._X[...,:3], p._X[...,:3].copy()))
    V = np.vstack((s._X[...,3:], p._X[...,3:].copy()))
    integrator = LeapfrogIntegrator(lm10_potential._acceleration_at,
                                    np.array(X), np.array(V),
                                    args=(X.shape[0], np.zeros_like(X)))
    ts, rs, vs = integrator.run(t1=simulation.t1, t2=simulation.t2, dt=-1)

    s_orbit = np.vstack((rs[:,0][:,np.newaxis].T, vs[:,0][:,np.newaxis].T)).T
    p_orbits = np.vstack((rs[:,1:].T, vs[:,1:].T)).T
    t_idx = np.array([np.argmin(np.fabs(ts - t)) for t in p.tub])

    p_x = np.array([p_orbits[jj,ii] for ii,jj in enumerate(t_idx)])
    s_x = np.array([s_orbit[jj,0] for jj in t_idx])
    diff = p_x-s_x

    norm_r = s_x[:,:3] / np.sqrt(np.sum(s_x[:,:3]**2, axis=-1))[:,np.newaxis]
    norm_diff_r = diff[:,:3] / np.sqrt(np.sum(diff[:,:3]**2, axis=-1))[:,np.newaxis]
    dot_prod_r = np.sum(norm_diff_r*norm_r, axis=-1)
    tail_bit = (dot_prod_r > 0.).astype(int)*2 - 1
    ################################################################

    satellite = satellite.to_frame(heliocentric)
    particles = particles.to_frame(heliocentric)

    # observe the particles if necessary
    if particle_error_model is not None:
        logger.info("Observing particles with {}".format(particle_error_model))
        particle_errors = particle_error_model(particles)
        o_particles = particles.observe(particle_errors)
    else:
        logger.info("Not observing particles")
        o_particles = particles

    # observe the satellite if necessary
    if satellite_error_model:
        logger.info("Observing satellite with {}".format(satellite_error_model))
        satellite_errors = satellite_error_model(satellite)
        o_satellite = satellite.observe(satellite_errors)
    else:
        logger.info("Not observing satellite")
        o_satellite = satellite

    # make a plot of true and observed positions
    true_gc_particles = particles.to_frame(galactocentric)
    gc_particles = o_particles.to_frame(galactocentric)
    all_gc_particles = simulation.particles(N=1000, expr="tub!=0")\
                                 .to_frame(galactocentric)

    fig,axes = plt.subplots(1,2,figsize=(16,8))
    markersize = 6.
    axes[0].plot(all_gc_particles["x"].value, all_gc_particles["z"].value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.25)
    axes[0].plot(true_gc_particles["x"].value, true_gc_particles["z"].value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.5)
    axes[0].plot(gc_particles["x"].value, gc_particles["z"].value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.5, c='#ca0020')

    axes[1].plot(all_gc_particles["vx"].to(u.km/u.s).value,
                 all_gc_particles["vz"].to(u.km/u.s).value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.25)
    axes[1].plot(true_gc_particles["vx"].to(u.km/u.s).value,
                 true_gc_particles["vz"].to(u.km/u.s).value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.5)
    axes[1].plot(gc_particles["vx"].to(u.km/u.s).value,
                 gc_particles["vz"].to(u.km/u.s).value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.5, c='#ca0020')

    fname = os.path.splitext(os.path.basename(output_file))[0]
    fig.savefig(os.path.join(os.path.split(output_file)[0],
                             "{}.{}".format(fname, 'png')))

    with h5py.File(output_file, "w") as f:
        # add particle positions to file
        grp = f.create_group("particles")
        grp["data"] = o_particles._repr_X

        if isinstance(o_particles, ObservedParticle):
            grp["error"] = o_particles._repr_error_X
            grp["true_data"] = particles._repr_X
        grp["coordinate_names"] = o_particles.frame.coord_names
        grp["units"] = [str(x) for x in o_particles._repr_units]
        grp["tub"] = o_particles.tub
        grp["tail_bit"] = tail_bit

        grp = f.create_group("satellite")
        grp["data"] = o_satellite._repr_X
        if isinstance(o_satellite, ObservedParticle):
            grp["error"] = o_satellite._repr_error_X
            grp["true_data"] = satellite._repr_X
        grp["coordinate_names"] = o_satellite.frame.coord_names
        grp["units"] = [str(x) for x in o_satellite._repr_units]
        grp["m"] = o_satellite.m
        grp["v_disp"] = o_satellite.v_disp

        grp = f.create_group("simulation")
        grp["t1"] = simulation.t1
        grp["t2"] = simulation.t2

if __name__ == "__main__":
    from argparse import ArgumentParser

    parser = ArgumentParser(description="")
    parser.add_argument("-v", "--verbose", action="store_true",
                        dest="verbose", default=False,
                        help="Be chatty! (default = False)")
    parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
                    default=False, help="Be quiet! (default = False)")
    parser.add_argument("-o", "--overwrite", dest="overwrite", default=False, action="store_true",
                        help="Overwrite any existing data.")

    parser.add_argument("-f", "--file", dest="output_file", required=True,
                        help="Name of the file to store the data.")
    parser.add_argument("--class_name", dest="class_name", required=True,
                        help="Name of the simulation data class, e.g. SgrSimulation")
    parser.add_argument("--expr", dest="expr", default=None, type=str,
                        help="Selection expression (for numexpr) for simulation particles.")
    parser.add_argument("--N", dest="N", default=None, type=int,
                        help="Number of particles.")
    parser.add_argument("--seed", dest="seed", default=None, type=int,
                        help="Seed for random number generator.")

    parser.add_argument("--mass", dest="mass", required=True, type=str,
                        help="Satellite mass.")

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    elif args.quiet:
        logging.basicConfig(level=logging.ERROR)
    else:
        logging.basicConfig(level=logging.INFO)

    """
        e.g.:

python scripts/observe_simulation.py -v --class_name=SgrSimulation --expr='tub!=0' \
--N=256 --file=/Users/adrian/Projects/streams/data/observed_particles/2.5e6_N1024.hdf5 \
--seed=42 --mass="2.5e6" --overwrite

python scripts/observe_simulation.py -v --class_name=SgrSimulation --expr='tub!=0' \
--N=256 --file=/Users/adrian/projects/streams/data/observed_particles/2.5e7_N1024.hdf5 \
--seed=42 --mass="2.5e7" --overwrite

python scripts/observe_simulation.py -v --class_name=SgrSimulation --expr='tub!=0' \
--N=1024 --file=/Users/adrian/projects/streams/data/observed_particles/2.5e8_N1024.hdf5 \
--seed=42 --mass="2.5e8" --overwrite
    """

    # TODO: class kwargs
    # particle_error_model=gaia_spitzer_errors, satellite_error_model=gaia_spitzer_errors,
    # particle_error_model=gaia_spitzer_errors, satellite_error_model=None,
    observe_simulation(args.class_name,
        particle_error_model=gaia_spitzer_errors, satellite_error_model=gaia_spitzer_errors,
        selection_expr=args.expr, N=args.N, output_file=args.output_file,
        overwrite=args.overwrite, seed=args.seed, class_kwargs=dict(mass=args.mass))
<commit_msg>moved tailbit stuff and added mdot stuff<commit_after># coding: utf-8

""" Observe particles from a given simulation """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys
import copy
import logging
import random

# Third-party
import astropy.units as u
from astropy.utils.console import color_print
import emcee
import h5py
import matplotlib.pyplot as plt
import numpy as np
np.seterr(all="ignore")
import scipy
scipy.seterr(all="ignore")
import yaml

# Project
from streams import usys
from streams.coordinates.frame import heliocentric, galactocentric
from streams.dynamics import ObservedParticle
import streams.io as s_io
from streams.observation.gaia import gaia_spitzer_errors
from streams.potential.lm10 import LawMajewski2010
from streams.util import _parse_quantity
from streams.integrate import LeapfrogIntegrator

# Create logger
logger = logging.getLogger(__name__)

def observe_simulation(class_name, particle_error_model=None, satellite_error_model=None,
                       selection_expr=None, N=None, output_file=None, overwrite=False,
                       seed=None, class_kwargs=dict()):
    """ Observe simulation data and write the output to a standard HDF5 format.

        TODO: handle missing dimensions

        Parameters
        ----------

    """

    if os.path.exists(output_file) and overwrite:
        os.remove(output_file)

    if os.path.exists(output_file):
        raise IOError("File '{}' already exists! Did you "
                      "want to use overwrite=True?".format(output_file))

    # read the simulation data from the specified class
    if seed is None:
        seed = np.random.randint(100)

    logger.debug("Using seed: {}".format(seed))
    np.random.seed(seed)
    random.seed(seed)

    try:
        Simulation = getattr(s_io, class_name)
    except AttributeError:
        raise ValueError("Simulation class '{}' not found!".format(class_name))

    # instantiate the specified simulation class with any additional keyword
    #   arguments, e.g., mass from SgrSimulation
    simulation = Simulation(**class_kwargs)

    # read particles from the simulation class
    sim_time = simulation.particle_units[0]/simulation.particle_units[-1]

    # HACK HACK HACK
    selection_expr = "(tub!=0) & (tub>{}) & (tub<{}) & (sqrt((x+8)**2 + y**2 + z**2)<60)"\
                     .format((1400*u.Myr).to(sim_time).value,
                             (5300*u.Myr).to(sim_time).value)
    particles = simulation.particles(N=N, expr=selection_expr)

    logger.debug("Read in {} particles with expr='{}'"\
                 .format(particles.nparticles, selection_expr))

    # read the satellite position
    satellite = simulation.satellite()
    logger.debug("Read in present position of satellite...")

    satellite = satellite.to_frame(heliocentric)
    particles = particles.to_frame(heliocentric)

    # observe the particles if necessary
    if particle_error_model is not None:
        logger.info("Observing particles with {}".format(particle_error_model))
        particle_errors = particle_error_model(particles)
        o_particles = particles.observe(particle_errors)
    else:
        logger.info("Not observing particles")
        o_particles = particles

    # observe the satellite if necessary
    if satellite_error_model:
        logger.info("Observing satellite with {}".format(satellite_error_model))
        satellite_errors = satellite_error_model(satellite)
        o_satellite = satellite.observe(satellite_errors)
    else:
        logger.info("Not observing satellite")
        o_satellite = satellite

    # make a plot of true and observed positions
    true_gc_particles = particles.to_frame(galactocentric)
    gc_particles = o_particles.to_frame(galactocentric)
    all_gc_particles = simulation.particles(N=1000, expr="tub!=0")\
                                 .to_frame(galactocentric)

    fig,axes = plt.subplots(1,2,figsize=(16,8))
    markersize = 6.
    axes[0].plot(all_gc_particles["x"].value, all_gc_particles["z"].value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.25)
    axes[0].plot(true_gc_particles["x"].value, true_gc_particles["z"].value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.5)
    axes[0].plot(gc_particles["x"].value, gc_particles["z"].value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.5, c='#ca0020')

    axes[1].plot(all_gc_particles["vx"].to(u.km/u.s).value,
                 all_gc_particles["vz"].to(u.km/u.s).value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.25)
    axes[1].plot(true_gc_particles["vx"].to(u.km/u.s).value,
                 true_gc_particles["vz"].to(u.km/u.s).value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.5)
    axes[1].plot(gc_particles["vx"].to(u.km/u.s).value,
                 gc_particles["vz"].to(u.km/u.s).value,
                 markersize=markersize, marker='o', linestyle='none', alpha=0.5, c='#ca0020')

    fname = os.path.splitext(os.path.basename(output_file))[0]
    fig.savefig(os.path.join(os.path.split(output_file)[0],
                             "{}.{}".format(fname, 'png')))

    with h5py.File(output_file, "w") as f:
        # add particle positions to file
        grp = f.create_group("particles")
        grp["data"] = o_particles._repr_X

        if isinstance(o_particles, ObservedParticle):
            grp["error"] = o_particles._repr_error_X
            grp["true_data"] = particles._repr_X
        grp["coordinate_names"] = o_particles.frame.coord_names
        grp["units"] = [str(x) for x in o_particles._repr_units]
        grp["tub"] = o_particles.tub
        grp["tail_bit"] = o_particles.tail_bit

        grp = f.create_group("satellite")
        grp["data"] = o_satellite._repr_X
        if isinstance(o_satellite, ObservedParticle):
            grp["error"] = o_satellite._repr_error_X
            grp["true_data"] = satellite._repr_X
        grp["coordinate_names"] = o_satellite.frame.coord_names
        grp["units"] = [str(x) for x in o_satellite._repr_units]
        grp["m0"] = o_satellite.m
        grp["mdot"] = o_satellite.mdot

        grp = f.create_group("simulation")
        grp["t1"] = simulation.t1
        grp["t2"] = simulation.t2

if __name__ == "__main__":
    from argparse import ArgumentParser

    parser = ArgumentParser(description="")
    parser.add_argument("-v", "--verbose", action="store_true",
                        dest="verbose", default=False,
                        help="Be chatty! (default = False)")
    parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
                    default=False, help="Be quiet! (default = False)")
    parser.add_argument("-o", "--overwrite", dest="overwrite", default=False, action="store_true",
                        help="Overwrite any existing data.")

    parser.add_argument("-f", "--file", dest="output_file", required=True,
                        help="Name of the file to store the data.")
    parser.add_argument("--class_name", dest="class_name", required=True,
                        help="Name of the simulation data class, e.g. SgrSimulation")
    parser.add_argument("--expr", dest="expr", default=None, type=str,
                        help="Selection expression (for numexpr) for simulation particles.")
    parser.add_argument("--N", dest="N", default=None, type=int,
                        help="Number of particles.")
    parser.add_argument("--seed", dest="seed", default=None, type=int,
                        help="Seed for random number generator.")

    parser.add_argument("--mass", dest="mass", required=True, type=str,
                        help="Satellite mass.")

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    elif args.quiet:
        logging.basicConfig(level=logging.ERROR)
    else:
        logging.basicConfig(level=logging.INFO)

    """
        e.g.:

python scripts/observe_simulation.py -v --class_name=SgrSimulation --expr='tub!=0' \
--N=256 --file=/Users/adrian/Projects/streams/data/observed_particles/2.5e6_N1024.hdf5 \
--seed=42 --mass="2.5e6" --overwrite

python scripts/observe_simulation.py -v --class_name=SgrSimulation --expr='tub!=0' \
--N=256 --file=/Users/adrian/projects/streams/data/observed_particles/2.5e7_N1024.hdf5 \
--seed=42 --mass="2.5e7" --overwrite

python scripts/observe_simulation.py -v --class_name=SgrSimulation --expr='tub!=0' \
--N=1024 --file=/Users/adrian/projects/streams/data/observed_particles/2.5e8_N1024.hdf5 \
--seed=42 --mass="2.5e8" --overwrite
    """

    # TODO: class kwargs
    # particle_error_model=gaia_spitzer_errors, satellite_error_model=gaia_spitzer_errors,
    # particle_error_model=gaia_spitzer_errors, satellite_error_model=None,
    observe_simulation(args.class_name,
        particle_error_model=gaia_spitzer_errors, satellite_error_model=gaia_spitzer_errors,
        selection_expr=args.expr, N=args.N, output_file=args.output_file,
        overwrite=args.overwrite, seed=args.seed, class_kwargs=dict(mass=args.mass))
<|endoftext|>


<commit_before>from shakespearelang import Shakespeare
from pathlib import Path
import cProfile

path = Path(__file__).parent / f"shakespearelang/tests/sample_plays/sierpinski.spl"
with path.open() as f:
    cProfile.run("Shakespeare(f.read()).run()", "profilestats")
<commit_msg>Fix profiling script<commit_after>from shakespearelang import Shakespeare
from pathlib import Path
import cProfile

path = Path(__file__).parent.parent / f"shakespearelang/tests/sample_plays/sierpinski.spl"
with path.open() as f:
    cProfile.run("Shakespeare(f.read()).run()", "profilestats")
<|endoftext|>


<commit_before>import pandas as pd

def country_probabilities(df):
    user_age = df['age']
    user_gender = df['gender']

    gender_mask = bkts['gender'] == user_gender
    age_mask = (bkts['from_age'] <= user_age) & (user_age <= bkts['to_age'])
    similar_population = bkts.loc[age_mask & gender_mask]

    for index, row in similar_population.iterrows():
        country = row['country_destination']
        df['similar_population_in_' + country] = row['population_in_thousands'] # / country_population[country]

    return df

path = '../datasets/processed/'
train_users = pd.read_csv(path + 'processed_train_users.csv')
test_users = pd.read_csv(path + 'processed_test_users.csv')

path = '../datasets/raw/'
bkts = pd.read_csv(path + 'age_gender_bkts.csv')

users = pd.concat((train_users, test_users), axis=0, ignore_index=True)

bkts.loc[bkts['age_bucket'] == '100+', 'age_bucket'] = '100-200'

bkts['from_age'] = bkts.age_bucket.str.split('-').apply(pd.Series, 2)[0].astype(int)
bkts['to_age'] = bkts.age_bucket.str.split('-').apply(pd.Series, 2)[1].astype(int)

bkts.drop('age_bucket', axis=1, inplace=True)

bkts.loc[bkts['gender'] == 'male', 'gender'] = 'MALE'
bkts.loc[bkts['gender'] == 'female', 'gender'] = 'FEMALE'

country_population = bkts.groupby('country_destination')['population_in_thousands'].sum()

train_users = train_users.apply(country_probabilities, axis=1)
test_users = test_users.apply(country_probabilities, axis=1)

train_users.to_csv('train_users_extra.csv')
test_users.to_csv('test_users_extra.csv')
<commit_msg>Fix script<commit_after>import pandas as pd

def country_probabilities(df):
    user_age = df['age']
    user_gender = df['gender']

    gender_mask = bkts['gender'] == user_gender
    age_mask = (bkts['from_age'] <= user_age) & (user_age <= bkts['to_age'])
    similar_population = bkts.loc[age_mask & gender_mask]

    for index, row in similar_population.iterrows():
        country = row['country_destination']
        df['similar_population_in_' + country] = row['population_in_thousands'] # / country_population[country]

    return df

path = '../datasets/raw/'
train_users = pd.read_csv(path + 'train_users.csv')
test_users = pd.read_csv(path + 'test_users.csv')
bkts = pd.read_csv(path + 'age_gender_bkts.csv')

users = pd.concat((train_users, test_users), axis=0, ignore_index=True)

bkts.loc[bkts['age_bucket'] == '100+', 'age_bucket'] = '100-200'

bkts['from_age'] = bkts.age_bucket.str.split('-').apply(pd.Series, 2)[0].astype(int)
bkts['to_age'] = bkts.age_bucket.str.split('-').apply(pd.Series, 2)[1].astype(int)

bkts.drop('age_bucket', axis=1, inplace=True)

bkts.loc[bkts['gender'] == 'male', 'gender'] = 'MALE'
bkts.loc[bkts['gender'] == 'female', 'gender'] = 'FEMALE'

country_population = bkts.groupby('country_destination')['population_in_thousands'].sum()

train_users_exta = train_users.head().apply(country_probabilities, axis=1)
test_users_extra = test_users.head().apply(country_probabilities, axis=1)

similar_population_columns = test_users_extra.columns.difference(train_users.columns)
similar_population_columns = similar_population_columns.values

train_users_exta[similar_population_columns].to_csv('train_users_extra.csv')
train_users_exta[similar_population_columns].to_csv('test_users_extra.csv')

# TODO: Join
<|endoftext|>


<commit_before>data = []

def plot(x, y, format=None, label=None, lw=None):
    d = {
        "data": zip(x, y),
        }
    if label:
        d["label"] = label
    if format:
        options = parse_format(format)
        d.update(options)
    data.append(d)

def parse_format(f):
    assert f is not None
    options = {}
    if len(f) == 2:
        colors = {
                "k": "#000000",
                "r": "#FF0000",
                "g": "#00FF00",
                }
        if f[0] not in colors:
            raise Exception("Unsupported color")
        options["color"] = colors[f[0]]
        if f[1] == "-":
            options["points"] = {"show": False}
            options["lines"] = {"show": True}
        elif f[1] == "o":
            options["points"] = {"show": True}
            options["lines"] = {"show": False}
    return options

def show():
    from plotserver.runserver import main
    main()

def grid(*args, **kwargs):
    pass

def legend(*args, **kwargs):
    pass
<commit_msg>Implement lw, fix colors<commit_after>data = []

def plot(x, y, format=None, label=None, lw=None):
    d = {
        "data": zip(x, y),
        }
    if label:
        d["label"] = label
    if format:
        d.update(parse_format(format))
    if lw:
        lines = d.get("lines", {})
        lines["lineWidth"] = lw
        d["lines"] = lines
    data.append(d)

def parse_format(f):
    assert f is not None
    options = {}
    if len(f) == 2:
        colors = {
                "k": "#000000",
                "r": "#FF0000",
                "g": "#00FF00",
                "y": "#FFFF00",
                }
        if f[0] not in colors:
            raise Exception("Unsupported color: '%s'" % f[0])
        options["color"] = colors[f[0]]
        if f[1] == "-":
            options["points"] = {"show": False}
            options["lines"] = {"show": True}
        elif f[1] == "o":
            options["points"] = {"show": True}
            options["lines"] = {"show": False}
    return options

def show():
    from plotserver.runserver import main
    main()

def grid(*args, **kwargs):
    pass

def legend(*args, **kwargs):
    pass
<|endoftext|>


<commit_before># Copyright (c) 2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

from conary import trove, versions
from conary.deps import deps
from conary.lib import util
from conary.build import errors as builderrors
from conary.build import macros
from conary.build import use
from conary.build.recipe import Recipe, RECIPE_TYPE_REDIRECT

import itertools

class RedirectRecipe(Recipe):
    Flags = use.LocalFlags
    _recipeType = RECIPE_TYPE_REDIRECT
    internalAbstractBaseClass = 1

    def addRedirect(self, name, branchStr = None, sourceFlavor = None,
                    targetFlavor = None, fromTrove = None,
                    skipTargetMatching = False):
        if ((sourceFlavor is not None) and (targetFlavor is None)) or \
           ((targetFlavor is not None) and (sourceFlavor is None)):
            raise builderrors.RecipeFileError, \
                "sourceFlavor and targetFlavor must be specified jointly"

        if sourceFlavor is not None:
            f = deps.parseFlavor(sourceFlavor)
            if f is None:
                raise ValueError, 'invalid flavor %s' % sourceFlavor
            sourceFlavor = f

        if targetFlavor is not None:
            f = deps.parseFlavor(targetFlavor)
            if f is None:
                raise ValueError, 'invalid flavor %s' % targetFlavor
            targetFlavor = f

        if fromTrove is None:
            fromTrove = self.name
        elif fromTrove.find(":") != -1:
            raise ValueError, 'components cannot be individually redirected'

        if fromTrove.startswith("group-"):
            # how sad
            raise ValueError, "groups cannot be redirected"

        self.addTroveList.append((name, branchStr, sourceFlavor,
                                  targetFlavor, fromTrove, skipTargetMatching))

    def findTroves(self):
        self.size = 0

        validSize = True
        troveList = []

        packageSet = {}

        sourceSearch = {}
        fromRule = {}
        for (name, branchStr, sourceFlavor, targetFlavor,
             fromTrove, skipTargetMatching) in self.addTroveList:
            l = fromRule.setdefault(fromTrove, list())
            # the catch-all (with no sourceFlavor) has to be at the end
            if sourceFlavor is None:
                l.append((name, branchStr, sourceFlavor, targetFlavor,
                          skipTargetMatching))
            else:
                l.insert(0, (name, branchStr, sourceFlavor, targetFlavor,
                             skipTargetMatching))

            d = sourceSearch.setdefault(fromTrove, { self.branch : None })

        sourceTroveMatches = self.repos.getTroveLeavesByBranch(sourceSearch)

        if len(sourceTroveMatches) != len(sourceSearch):
            missing = set(sourceSearch) - set(sourceTroveMatches)
            raise builderrors.RecipeFileError, \
                    "No troves found with name(s) %s" % " ".join(missing)

        l = []
        for name, d in sourceTroveMatches.iteritems():
            for version, flavorList in d.iteritems():
                l += [ (name, (None, None), (version, x), True)
                                for x in flavorList ]

        trvCsDict = {}
        # We don't need to recurse here since we only support package
        # redirects
        cs = self.repos.createChangeSet(l, recurse = True, withFiles = False)
        for trvCs in cs.iterNewTroveList():
            info = (trvCs.getName(), trvCs.getNewVersion(),
                    trvCs.getNewFlavor())
            trvCsDict[info] = trvCs

        redirMap = {}

        names = sourceTroveMatches.keys()
        additionalNameQueue = util.IterableQueue()
        for name in itertools.chain(names, additionalNameQueue):
            versionDict = sourceTroveMatches.pop(name)

            destSet = fromRule.get(name, None)
            if destSet is None and ':' in name:
                # package redirections imply component redirections
                pkgName, compName = name.split(':')
                destSet = fromRule.get(pkgName, None)
                if destSet is not None:
                    destSet = set(
                        [ (x[0] + ':' + compName,) + x[1:] for x in destSet ])

            if destSet is None:
                raise builderrors.RecipeFileError, \
                    "Cannot find redirection for trove %s" % name

            # XXX the repository operations should be pulled out of all of
            # these loops
            additionalNames = set()
            for (destName, branchStr, sourceFlavorRestriction,
                targetFlavorRestriction, skipTargetMatching) in destSet:
                if branchStr[0] == '/':
                    branch = versions.VersionFromString(branchStr)
                    if not isinstance(branch, versions.Branch):
                        raise builderrors.RecipeFileError, \
                            "Redirects must specify branches or labels, " \
                            "not versions"

                    matches = self.repos.getTroveLeavesByBranch(
                                    { destName : { branch : None } })
                else:
                    label = versions.Label(branchStr)
                    matches = self.repos.getTroveLeavesByLabel(
                                    { destName : { label : None } })
                    # if there are multiple versions returned, the label
                    # may have matched multiple branches
                    if matches:
                        branches = set(x.branch() for x in matches[destName])
                        if len(branches) > 1:
                            raise builderrors.RecipeFileError, \
                                "Label %s matched multiple branches." % str(label)

                # use an empty list to indicate notexistance
                if destName not in matches:
                    # We're redirecting to something which doesn't
                    # exist. This is an error if it's the top of a
                    # redirect (a package), but generates an erase
                    # redirect if it's for a component.
                    if name in names:
                        raise builderrors.RecipeFileError, \
                            "Trove %s does not exist" % (destName)

                    redirMap[(name, sourceFlavor)] = (None, None, None, [])
                    continue

                # Get the flavors and branch available on the target
                targetFlavors = set()
                for version, flavorList in matches[destName].iteritems():
                    targetFlavors.update((version, x) for x in flavorList)
                del matches

                foundMatch = False
                for version, flavorList in versionDict.items():
                    for sourceFlavor in flavorList:
                        if sourceFlavorRestriction is not None and \
                           sourceFlavor != sourceFlavorRestriction: continue

                        match = None
                        for targetVersion, targetFlavor in targetFlavors:
                            if (not skipTargetMatching and
                                targetFlavorRestriction is not None and
                                targetFlavor != targetFlavorRestriction):
                                continue

                            if ((sourceFlavorRestriction is not None)
                                or skipTargetMatching
                                or sourceFlavor.score(targetFlavor) is not False):
                                match = targetVersion
                                break

                        if match is not None:
                            foundMatch = True
                            # we can redirect this trove. go ahead and
                            # setup the redirect for it, and add any troves
                            # it references to the todo list
                            trvCs = trvCsDict[(name, version, sourceFlavor)]

                            # we can't integrity check here because we got
                            # the trove w/o files
                            trv = trove.Trove(trvCs, skipIntegrityChecks = True)

                            for info in trv.iterTroveList(strongRefs = True):
                                assert(info[1] == version)
                                assert(info[2] == sourceFlavor)
                                additionalNames.add(info[0])
                                d = sourceTroveMatches.setdefault(info[0], {})
                                flavorList = d.setdefault(info[1], [])
                                flavorList.append(info[2])

                            assert((name, sourceFlavor) not in redirMap)
                            redirMap[(name, sourceFlavor)] = \
                                (destName, match.branch(),
                                 targetFlavorRestriction,
                                 [ x[0] for x in
                                    trv.iterTroveList(strongRefs = True) ] )
                        elif targetFlavorRestriction is not None:
                            raise builderrors.RecipeFileError, \
                                "Trove %s does not exist for flavor %s" \
                                % (name, targetFlavor)
                if not foundMatch:
                    raise builderrors.CookError(
                    "Could not find target with satisfying flavor"
                    " for redirect %s - either create a redirect"
                    " with targetFlavor and sourceFlavor set, or"
                    " create a redirect with skipTargetMatching = True" % name)

            for name in additionalNames:
                additionalNameQueue.add(name)

        self.redirections = redirMap

    def getRedirections(self):
        return self.redirections

    def __init__(self, repos, cfg, branch, flavor, extraMacros={}):
        self.repos = repos
        self.cfg = cfg
        self.redirections = {}
        self.branch = branch
        self.flavor = flavor
        self.addTroveList = []
        self.macros = macros.Macros()
        self.macros.update(extraMacros)

<commit_msg>source flavor for components of redirects wasn't set properly<commit_after># Copyright (c) 2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

from conary import trove, versions
from conary.deps import deps
from conary.lib import util
from conary.build import errors as builderrors
from conary.build import macros
from conary.build import use
from conary.build.recipe import Recipe, RECIPE_TYPE_REDIRECT

import itertools

class RedirectRecipe(Recipe):
    Flags = use.LocalFlags
    _recipeType = RECIPE_TYPE_REDIRECT
    internalAbstractBaseClass = 1

    def addRedirect(self, name, branchStr = None, sourceFlavor = None,
                    targetFlavor = None, fromTrove = None,
                    skipTargetMatching = False):
        if ((sourceFlavor is not None) and (targetFlavor is None)) or \
           ((targetFlavor is not None) and (sourceFlavor is None)):
            raise builderrors.RecipeFileError, \
                "sourceFlavor and targetFlavor must be specified jointly"

        if sourceFlavor is not None:
            f = deps.parseFlavor(sourceFlavor)
            if f is None:
                raise ValueError, 'invalid flavor %s' % sourceFlavor
            sourceFlavor = f

        if targetFlavor is not None:
            f = deps.parseFlavor(targetFlavor)
            if f is None:
                raise ValueError, 'invalid flavor %s' % targetFlavor
            targetFlavor = f

        if fromTrove is None:
            fromTrove = self.name
        elif fromTrove.find(":") != -1:
            raise ValueError, 'components cannot be individually redirected'

        if fromTrove.startswith("group-"):
            # how sad
            raise ValueError, "groups cannot be redirected"

        self.addTroveList.append((name, branchStr, sourceFlavor,
                                  targetFlavor, fromTrove, skipTargetMatching))

    def findTroves(self):
        self.size = 0

        validSize = True
        troveList = []

        packageSet = {}

        sourceSearch = {}
        fromRule = {}
        for (name, branchStr, sourceFlavor, targetFlavor,
             fromTrove, skipTargetMatching) in self.addTroveList:
            l = fromRule.setdefault(fromTrove, list())
            # the catch-all (with no sourceFlavor) has to be at the end
            if sourceFlavor is None:
                l.append((name, branchStr, sourceFlavor, targetFlavor,
                          skipTargetMatching))
            else:
                l.insert(0, (name, branchStr, sourceFlavor, targetFlavor,
                             skipTargetMatching))

            d = sourceSearch.setdefault(fromTrove, { self.branch : None })

        sourceTroveMatches = self.repos.getTroveLeavesByBranch(sourceSearch)

        if len(sourceTroveMatches) != len(sourceSearch):
            missing = set(sourceSearch) - set(sourceTroveMatches)
            raise builderrors.RecipeFileError, \
                    "No troves found with name(s) %s" % " ".join(missing)

        l = []
        for name, d in sourceTroveMatches.iteritems():
            for version, flavorList in d.iteritems():
                l += [ (name, (None, None), (version, x), True)
                                for x in flavorList ]

        trvCsDict = {}
        # We don't need to recurse here since we only support package
        # redirects
        cs = self.repos.createChangeSet(l, recurse = True, withFiles = False)
        for trvCs in cs.iterNewTroveList():
            info = (trvCs.getName(), trvCs.getNewVersion(),
                    trvCs.getNewFlavor())
            trvCsDict[info] = trvCs

        redirMap = {}

        names = sourceTroveMatches.keys()
        additionalNameQueue = util.IterableQueue()
        for name in itertools.chain(names, additionalNameQueue):
            versionDict = sourceTroveMatches.pop(name)

            destSet = fromRule.get(name, None)
            if destSet is None and ':' in name:
                # package redirections imply component redirections
                pkgName, compName = name.split(':')
                destSet = fromRule.get(pkgName, None)
                if destSet is not None:
                    destSet = set(
                        [ (x[0] + ':' + compName,) + x[1:] for x in destSet ])

            if destSet is None:
                raise builderrors.RecipeFileError, \
                    "Cannot find redirection for trove %s" % name

            # XXX the repository operations should be pulled out of all of
            # these loops
            additionalNames = set()
            for (destName, branchStr, sourceFlavorRestriction,
                targetFlavorRestriction, skipTargetMatching) in destSet:
                if branchStr[0] == '/':
                    branch = versions.VersionFromString(branchStr)
                    if not isinstance(branch, versions.Branch):
                        raise builderrors.RecipeFileError, \
                            "Redirects must specify branches or labels, " \
                            "not versions"

                    matches = self.repos.getTroveLeavesByBranch(
                                    { destName : { branch : None } })
                else:
                    label = versions.Label(branchStr)
                    matches = self.repos.getTroveLeavesByLabel(
                                    { destName : { label : None } })
                    # if there are multiple versions returned, the label
                    # may have matched multiple branches
                    if matches:
                        branches = set(x.branch() for x in matches[destName])
                        if len(branches) > 1:
                            raise builderrors.RecipeFileError, \
                                "Label %s matched multiple branches." % str(label)

                targetFlavors = set()
                if destName not in matches:
                    # We're redirecting to something which doesn't
                    # exist. This is an error if it's the top of a
                    # redirect (a package), but generates an erase
                    # redirect if it's for a component.
                    if name in names:
                        raise builderrors.RecipeFileError, \
                            "Trove %s does not exist" % (destName)
                else:
                    # Get the flavors and branch available on the target
                    for version, flavorList in matches[destName].iteritems():
                        targetFlavors.update((version, x) for x in flavorList)
                del matches

                foundMatch = False
                for version, flavorList in versionDict.items():
                    for sourceFlavor in flavorList:
                        if sourceFlavorRestriction is not None and \
                           sourceFlavor != sourceFlavorRestriction: continue

                        match = None
                        for targetVersion, targetFlavor in targetFlavors:
                            if (not skipTargetMatching and
                                targetFlavorRestriction is not None and
                                targetFlavor != targetFlavorRestriction):
                                continue

                            if ((sourceFlavorRestriction is not None)
                                or skipTargetMatching
                                or sourceFlavor.score(targetFlavor) is not False):
                                match = targetVersion
                                break

                        if match is not None:
                            foundMatch = True
                            # we can redirect this trove. go ahead and
                            # setup the redirect for it, and add any troves
                            # it references to the todo list
                            trvCs = trvCsDict[(name, version, sourceFlavor)]

                            # we can't integrity check here because we got
                            # the trove w/o files
                            trv = trove.Trove(trvCs, skipIntegrityChecks = True)

                            for info in trv.iterTroveList(strongRefs = True):
                                assert(info[1] == version)
                                assert(info[2] == sourceFlavor)
                                additionalNames.add(info[0])
                                d = sourceTroveMatches.setdefault(info[0], {})
                                flavorList = d.setdefault(info[1], [])
                                flavorList.append(info[2])

                            assert((name, sourceFlavor) not in redirMap)
                            redirMap[(name, sourceFlavor)] = \
                                (destName, match.branch(),
                                 targetFlavorRestriction,
                                 [ x[0] for x in
                                    trv.iterTroveList(strongRefs = True) ] )
                        elif not targetFlavors:
                            # redirect to nothing
                            foundMatch = True
                            redirMap[(name, sourceFlavor)] = \
                                                (None, None, None, [])
                        elif targetFlavorRestriction is not None:
                            raise builderrors.RecipeFileError, \
                                "Trove %s does not exist for flavor %s" \
                                % (name, targetFlavor)

                if not foundMatch:
                    raise builderrors.CookError(
                    "Could not find target with satisfying flavor"
                    " for redirect %s - either create a redirect"
                    " with targetFlavor and sourceFlavor set, or"
                    " create a redirect with skipTargetMatching = True" % name)

            for name in additionalNames:
                additionalNameQueue.add(name)

        self.redirections = redirMap

    def getRedirections(self):
        return self.redirections

    def __init__(self, repos, cfg, branch, flavor, extraMacros={}):
        self.repos = repos
        self.cfg = cfg
        self.redirections = {}
        self.branch = branch
        self.flavor = flavor
        self.addTroveList = []
        self.macros = macros.Macros()
        self.macros.update(extraMacros)

<|endoftext|>


<commit_before>def driver(*args):
    '''
    the first argument is for the spectra stack normalization, and it can be one of these three options: 'Ha', 'Hb', 'UV', 'none'

    the second argument shows which line you want to fit a gaussian to: 'Ha', 'Hb', 'oiii', 'oii'
    if you want a second line, you add another argument.
    Example:
    > python3 stack_driver.py 'Ha' 'Ha' 'Hb' #normilize the spectra to Ha luminosity, stack it, and then measure Ha and Hb lines.

    modify the paths and the criteria for objects to use in the stacks below.
    '''
    import subprocess
    import sys
    import os
    import numpy as np
    from astropy.io import fits
    import time
    t00=time.time()

    #paths and source files
    specpath='/Path/Data/'
    tblpath='/Path/mosdef0d.fits'
    outfile='/Path/hanhb_normtouv.fits'
    lineoutfile='/Path/linefit_hanhb_normtouv.txt'

    #objects to stack
    tbl=fits.getdata(tblpath, ext=1)
    obj_ind=((tbl['z_mosfire'] < 2.) & (tbl['ha6565_lum'] != -999.))
    print('Stacking ',tbl['o32'][obj_ind].size,' objs')

    obj_ind=obj_ind*1. #convert boolean to 0,1 array

    def writetxt(filename, x):
        with open(filename,'w') as f:
            for a in x: print(a, file=f)

    writetxt("obj_ind.txt",obj_ind)
    objfile="obj_ind.txt"

    l="python3 /PathToCode/stackspec_script.py "+specpath+' '+tblpath+' '+objfile+' '+outfile

    if sys.argv[1] == 'Ha': os.system(l+' "Ha" > '+lineoutfile)
    if sys.argv[1] == 'Hb': os.system(l+' "Hb" > '+lineoutfile)
    if sys.argv[1] == 'UV': os.system(l+' "UV" > '+lineoutfile)
    if sys.argv[1] == 'none': os.system(l+' "none" > '+lineoutfile)

    l="python3 /PathToCode/linefit.py "+outfile
    if len(sys.argv) == 3:
        if sys.argv[2] == 'Ha': os.system(l+' "Ha" >> '+lineoutfile)
        if sys.argv[2] == 'Hb': os.system(l+' "Hb" >> '+lineoutfile)
        if sys.argv[2] == 'oiii': os.system(l+' "oiii" >> '+lineoutfile)
        if sys.argv[2] == 'oii': os.system(l+' "oii" >> '+lineoutfile)

    if len(sys.argv) == 4:
        if (sys.argv[2] == 'Ha') & (sys.argv[3] == 'Hb'): os.system(l+' "Ha" "Hb" >> '+lineoutfile)
        if (sys.argv[2] == 'Ha') & (sys.argv[3] == 'oiii'): os.system(l+' "Ha" "oiii" >> '+lineoutfile)
        if (sys.argv[2] == 'Ha') & (sys.argv[3] == 'oii'): os.system(l+' "Ha" "oii" >> '+lineoutfile)
        if (sys.argv[2] == 'Hb') & (sys.argv[3] == 'oiii'): os.system(l+' "Hb" "oiii" >> '+lineoutfile)
        if (sys.argv[2] == 'Hb') & (sys.argv[3] == 'oii'): os.system(l+' "Hb" "oii" >> '+lineoutfile)
        if (sys.argv[2] == 'oiii') & (sys.argv[3] == 'oii'): os.system(l+' "oiii" "oii" >> '+lineoutfile)

    l="rm "+objfile
    os.system(l)
    print('The program took: {0:3.2f}'.format((time.time()-t00)/60.),' min.')
    print('~~~~ Bye!! ~~~~')

driver()
<commit_msg>Added print function compatibility for Python 2<commit_after>from __future__ import print_function

def driver(*args):
    '''
    the first argument is for the spectra stack normalization, and it can be one of these three options: 'Ha', 'Hb', 'UV', 'none'

    the second argument shows which line you want to fit a gaussian to: 'Ha', 'Hb', 'oiii', 'oii'
    if you want a second line, you add another argument.
    Example:
    > python3 stack_driver.py 'Ha' 'Ha' 'Hb' #normilize the spectra to Ha luminosity, stack it, and then measure Ha and Hb lines.

    modify the paths and the criteria for objects to use in the stacks below.
    '''
    import subprocess
    import sys
    import os
    import numpy as np
    from astropy.io import fits
    import time
    t00=time.time()

    #paths and source files
    specpath='/Path/Data/'
    tblpath='/Path/mosdef0d.fits'
    outfile='/Path/hanhb_normtouv.fits'
    lineoutfile='/Path/linefit_hanhb_normtouv.txt'

    #objects to stack
    tbl=fits.getdata(tblpath, ext=1)
    obj_ind=((tbl['z_mosfire'] < 2.) & (tbl['ha6565_lum'] != -999.))
    print('Stacking ',tbl['o32'][obj_ind].size,' objs')

    obj_ind=obj_ind*1. #convert boolean to 0,1 array

    def writetxt(filename, x):
        with open(filename,'w') as f:
            for a in x: print(a, file=f)

    writetxt("obj_ind.txt",obj_ind)
    objfile="obj_ind.txt"

    l="python3 /PathToCode/stackspec_script.py "+specpath+' '+tblpath+' '+objfile+' '+outfile

    if sys.argv[1] == 'Ha': os.system(l+' "Ha" > '+lineoutfile)
    if sys.argv[1] == 'Hb': os.system(l+' "Hb" > '+lineoutfile)
    if sys.argv[1] == 'UV': os.system(l+' "UV" > '+lineoutfile)
    if sys.argv[1] == 'none': os.system(l+' "none" > '+lineoutfile)

    l="python3 /PathToCode/linefit.py "+outfile
    if len(sys.argv) == 3:
        if sys.argv[2] == 'Ha': os.system(l+' "Ha" >> '+lineoutfile)
        if sys.argv[2] == 'Hb': os.system(l+' "Hb" >> '+lineoutfile)
        if sys.argv[2] == 'oiii': os.system(l+' "oiii" >> '+lineoutfile)
        if sys.argv[2] == 'oii': os.system(l+' "oii" >> '+lineoutfile)

    if len(sys.argv) == 4:
        if (sys.argv[2] == 'Ha') & (sys.argv[3] == 'Hb'): os.system(l+' "Ha" "Hb" >> '+lineoutfile)
        if (sys.argv[2] == 'Ha') & (sys.argv[3] == 'oiii'): os.system(l+' "Ha" "oiii" >> '+lineoutfile)
        if (sys.argv[2] == 'Ha') & (sys.argv[3] == 'oii'): os.system(l+' "Ha" "oii" >> '+lineoutfile)
        if (sys.argv[2] == 'Hb') & (sys.argv[3] == 'oiii'): os.system(l+' "Hb" "oiii" >> '+lineoutfile)
        if (sys.argv[2] == 'Hb') & (sys.argv[3] == 'oii'): os.system(l+' "Hb" "oii" >> '+lineoutfile)
        if (sys.argv[2] == 'oiii') & (sys.argv[3] == 'oii'): os.system(l+' "oiii" "oii" >> '+lineoutfile)

    l="rm "+objfile
    os.system(l)
    print('The program took: {0:3.2f}'.format((time.time()-t00)/60.),' min.')
    print('~~~~ Bye!! ~~~~')

driver()
<|endoftext|>