<commit_before>#!/usr/bin/env python
"""Script to load part files into a sql database.

For the inherent alpha model

We assume that the host temperature is well known so we will restrict the
results to the host temperature. (within error bars)
For HD30501 and HD211847 this means  +- 50K so a fixed temperature.

"""
import argparse
import os
import sys

import sqlalchemy as sa

import simulators
from bin.coadd_analysis_module import (chi2_parabola_plots, compare_spectra,
                                       contours, display_arbitary_norm_values,
                                       fix_host_parameters,
                                       fix_host_parameters_reduced_gamma,
                                       get_column_limits, get_npix_values,
                                       parabola_plots, rv_plot,
                                       smallest_chi2_values, test_figure)
from mingle.utilities.param_file import get_host_params
from mingle.utilities.phoenix_utils import closest_model_params

def parse_args(args):
    """Take care of all the argparse stuff.

    :returns: the args
    """
    parser = argparse.ArgumentParser(description='Chisquare analysis.')
    parser.add_argument('star', help='Star Name')
    parser.add_argument('obsnum', help="Observation label")
    parser.add_argument('--suffix', default=None,
                        help='Suffix to add to database name.')
    parser.add_argument("-e", "--echo", action="store_true",
                        help="Echo the SQL queries")
    parser.add_argument('-v', '--verbose', action="store_true",
                        help='Turn on Verbose.')
    parser.add_argument('-p', '--npars', type=int, default=3,
                        help='Number of interesting parameters. (default=3)')
    parser.add_argument("-m", "--mode", default="parabola",
                        help="Analysis mode to choose",
                        choices=["parabola", "fixed_host_params", "param_limits",
                                 "smallest_chi2", "test", "contour", "arbnorm",
                                 "all", "rvplot", "chi2_parabola", "compare_spectra"])
    return parser.parse_args(args)

def decompose_database_name(database):
    """Database names of form */Star_obsnum_chip...db."""
    os.path.split(database)
    path, name = os.path.split(database)
    name_split = name.split("_")
    star, obsnum = name_split[0].split("-")
    chip = name_split[1]
    return path, star, obsnum, chip

def load_sql_table(database, name="chi2_table", echo=False, verbose=False):
    sqlite_db = 'sqlite:///{0}'.format(database)
    try:
        engine = sa.create_engine(sqlite_db, echo=echo)
        table_names = engine.table_names()
    except Exception as e:
        print("\nAccessing sqlite_db = {0}\n".format(sqlite_db))
        print("cwd =", os.getcwd())
        raise e
    if verbose:
        print("Table names in database =", engine.table_names())
    if len(table_names) == 1:
        tb_name = table_names[0]
    else:
        raise ValueError("Database has two many tables {0}".format(table_names))
    if tb_name != name:
        raise NameError("Name given does not match table in database.")

    meta = sa.MetaData(bind=engine)
    db_table = sa.Table(name, meta, autoload=True)
    return db_table

def main(star, obsnum, suffix=None, echo=False, mode="parabola",
         verbose=False, npars=3):
    star = star.upper()
    suffix = "" if suffix is None else suffix
    database = os.path.join(
        simulators.paths["output_dir"], star,
        "{0}-{1}_coadd_iam_chisqr_results{2}.db".format(star, obsnum, suffix))

    if verbose:
        print("Database name ", database)
        print("Database exists", os.path.isfile(database))
    if not os.path.isfile(database):
        raise IOError("Database '{0}' does not exist.".format(database))

    path, dbstar, db_obsnum, chip = decompose_database_name(database)
    assert dbstar == star, "{} == {}".format(dbstar, star)
    assert db_obsnum == obsnum, "{} == {}".format(db_obsnum, obsnum)
    assert chip == "coadd", "{} == {}".format(chip, "coadd")

    os.makedirs(os.path.join(path, "plots"), exist_ok=True)  # make dir for plots

    teff, logg, fe_h = closest_model_params(*get_host_params(star))
    params = {"path": path, "star": star, "obsnum": obsnum, "chip": chip, "suffix": suffix,
              "teff": int(teff), "logg": float(logg), "fe_h": float(fe_h), "npars": npars}

    db_table = load_sql_table(database, verbose=verbose, echo=echo)

    # Put pixel counts in params
    params["npix"] = get_npix_values(db_table)

    print("Mode =", mode)

    if mode == "fixed_host_params":
        fix_host_parameters_reduced_gamma(db_table, params)
        fix_host_parameters(db_table, params)
    elif mode == "param_limits":
        get_column_limits(db_table, params)
    elif mode == "parabola":
        parabola_plots(db_table, params)
    elif mode == "smallest_chi2":
        smallest_chi2_values(db_table, params)
    elif mode == "contour":
        contours(db_table, params)
    elif mode == "test":
        test_figure(db_table, params)
    elif mode == "rvplot":
        rv_plot(db_table, params)
    elif mode == "arbnorm":
        display_arbitary_norm_values(db_table, params)
    elif mode == "chi2_parabola":
        chi2_parabola_plots(db_table, params)
    elif mode == "compare_spectra":
        compare_spectra(db_table, params)
    elif mode == "all":
        fix_host_parameters_reduced_gamma(db_table, params)
        get_column_limits(db_table, params)
        fix_host_parameters(db_table, params)
        display_arbitary_norm_values(db_table, params)
        smallest_chi2_values(db_table, params)
        parabola_plots(db_table, params)
        contours(db_table, params)
        test_figure(db_table, params)
        chi2_parabola_plots(db_table, params)
        compare_spectra(db_table, params)
    print("Done")
    return 0

if __name__ == '__main__':
    args = vars(parse_args(sys.argv[1:]))
    opts = {k: args[k] for k in args}

    sys.exit(main(**opts))
<commit_msg>change warning output<commit_after>#!/usr/bin/env python
"""Script to load part files into a sql database.

For the inherent alpha model

We assume that the host temperature is well known so we will restrict the
results to the host temperature. (within error bars)
For HD30501 and HD211847 this means  +- 50K so a fixed temperature.

"""
import argparse
import os
import sys

import sqlalchemy as sa

import simulators
from bin.coadd_analysis_module import (chi2_parabola_plots, compare_spectra,
                                       contours, display_arbitary_norm_values,
                                       fix_host_parameters,
                                       fix_host_parameters_reduced_gamma,
                                       get_column_limits, get_npix_values,
                                       parabola_plots, rv_plot,
                                       smallest_chi2_values, test_figure)
from mingle.utilities.param_file import get_host_params
from mingle.utilities.phoenix_utils import closest_model_params

def parse_args(args):
    """Take care of all the argparse stuff.

    :returns: the args
    """
    parser = argparse.ArgumentParser(description='Chisquare analysis.')
    parser.add_argument('star', help='Star Name')
    parser.add_argument('obsnum', help="Observation label")
    parser.add_argument('--suffix', default=None,
                        help='Suffix to add to database name.')
    parser.add_argument("-e", "--echo", action="store_true",
                        help="Echo the SQL queries")
    parser.add_argument('-v', '--verbose', action="store_true",
                        help='Turn on Verbose.')
    parser.add_argument('-p', '--npars', type=int, default=3,
                        help='Number of interesting parameters. (default=3)')
    parser.add_argument("-m", "--mode", default="parabola",
                        help="Analysis mode to choose",
                        choices=["parabola", "fixed_host_params", "param_limits",
                                 "smallest_chi2", "test", "contour", "arbnorm",
                                 "all", "rvplot", "chi2_parabola", "compare_spectra"])
    return parser.parse_args(args)

def decompose_database_name(database):
    """Database names of form */Star_obsnum_chip...db."""
    os.path.split(database)
    path, name = os.path.split(database)
    name_split = name.split("_")
    star, obsnum = name_split[0].split("-")
    chip = name_split[1]
    return path, star, obsnum, chip

def load_sql_table(database, name="chi2_table", echo=False, verbose=False):
    sqlite_db = 'sqlite:///{0}'.format(database)
    try:
        engine = sa.create_engine(sqlite_db, echo=echo)
        table_names = engine.table_names()
    except Exception as e:
        print("\nAccessing sqlite_db = {0}\n".format(sqlite_db))
        print("cwd =", os.getcwd())
        raise e
    if verbose:
        print("Table names in database =", engine.table_names())
    if len(table_names) == 1:
        tb_name = table_names[0]
    else:
        raise ValueError("Database does not just have 1 table. {0}, len={1}".format(table_names. len(table_names)))
    if tb_name != name:
        raise NameError("Name {0} given does not match table in database, {1}.".format(tb_name, table_names))

    meta = sa.MetaData(bind=engine)
    db_table = sa.Table(name, meta, autoload=True)
    return db_table

def main(star, obsnum, suffix=None, echo=False, mode="parabola",
         verbose=False, npars=3):
    star = star.upper()
    suffix = "" if suffix is None else suffix
    database = os.path.join(
        simulators.paths["output_dir"], star,
        "{0}-{1}_coadd_iam_chisqr_results{2}.db".format(star, obsnum, suffix))

    if verbose:
        print("Database name ", database)
        print("Database exists", os.path.isfile(database))
    if not os.path.isfile(database):
        raise IOError("Database '{0}' does not exist.".format(database))

    path, dbstar, db_obsnum, chip = decompose_database_name(database)
    assert dbstar == star, "{} == {}".format(dbstar, star)
    assert db_obsnum == obsnum, "{} == {}".format(db_obsnum, obsnum)
    assert chip == "coadd", "{} == {}".format(chip, "coadd")

    os.makedirs(os.path.join(path, "plots"), exist_ok=True)  # make dir for plots

    teff, logg, fe_h = closest_model_params(*get_host_params(star))
    params = {"path": path, "star": star, "obsnum": obsnum, "chip": chip, "suffix": suffix,
              "teff": int(teff), "logg": float(logg), "fe_h": float(fe_h), "npars": npars}

    db_table = load_sql_table(database, verbose=verbose, echo=echo)

    # Put pixel counts in params
    params["npix"] = get_npix_values(db_table)

    print("Mode =", mode)

    if mode == "fixed_host_params":
        fix_host_parameters_reduced_gamma(db_table, params)
        fix_host_parameters(db_table, params)
    elif mode == "param_limits":
        get_column_limits(db_table, params)
    elif mode == "parabola":
        parabola_plots(db_table, params)
    elif mode == "smallest_chi2":
        smallest_chi2_values(db_table, params)
    elif mode == "contour":
        contours(db_table, params)
    elif mode == "test":
        test_figure(db_table, params)
    elif mode == "rvplot":
        rv_plot(db_table, params)
    elif mode == "arbnorm":
        display_arbitary_norm_values(db_table, params)
    elif mode == "chi2_parabola":
        chi2_parabola_plots(db_table, params)
    elif mode == "compare_spectra":
        compare_spectra(db_table, params)
    elif mode == "all":
        fix_host_parameters_reduced_gamma(db_table, params)
        get_column_limits(db_table, params)
        fix_host_parameters(db_table, params)
        display_arbitary_norm_values(db_table, params)
        smallest_chi2_values(db_table, params)
        parabola_plots(db_table, params)
        contours(db_table, params)
        test_figure(db_table, params)
        chi2_parabola_plots(db_table, params)
        compare_spectra(db_table, params)
    print("Done")
    return 0

if __name__ == '__main__':
    args = vars(parse_args(sys.argv[1:]))
    opts = {k: args[k] for k in args}

    sys.exit(main(**opts))
<|endoftext|>


<commit_before>#coding: utf-8
import numpy as np
from scipy import optimize
from matplotlib import pyplot
from sklearn.datasets import load_digits, fetch_mldata
from sklearn.preprocessing import LabelBinarizer
from sklearn.metrics import confusion_matrix, classification_report

"""
正則化多層パーセプトロンによる手書き文字認識
・バッチ処理
・共役勾配法（Conjugate Gradient Method）によるパラメータ最適化
"""

def displayData(X):
    """
    データからランダムに100サンプル選んで可視化
    画像データは8x8ピクセルを仮定
    """
    # ランダムに100サンプル選ぶ
    sel = np.random.permutation(X.shape[0])
    sel = sel[:100]
    X = X[sel, :]
    for index, data in enumerate(X):
        pyplot.subplot(10, 10, index + 1)
        pyplot.axis('off')
        image = data.reshape((28, 28))
        pyplot.imshow(image, cmap=pyplot.cm.gray_r,
                     interpolation='nearest')
    pyplot.show()

def randInitializeWeights(L_in, L_out):
    """
    (-epsilon_init, +epsilon_init) の範囲で
    重みをランダムに初期化した重み行列を返す
    """
    # 入力となる層にはバイアス項が入るので+1が必要なので注意
    epsilon_init = 0.12
    W = np.random.rand(L_out, 1 + L_in) * 2 * epsilon_init - epsilon_init
    return W

def sigmoid(z):
    return 1.0 / (1 + np.exp(-z))

def sigmoidGradient(z):
    return sigmoid(z) * (1 - sigmoid(z))

def safe_log(x, minval=0.0000000001):
    return np.log(x.clip(min=minval))

def nnCostFunction(nn_params, *args):
    """NNのコスト関数とその偏微分を求める"""
    in_size, hid_size, num_labels, X, y, lam = args

    # ニューラルネットの全パラメータを行列形式に復元
    Theta1 = nn_params[0:(in_size + 1) * hid_size].reshape((hid_size, in_size + 1))
    Theta2 = nn_params[(in_size + 1) * hid_size:].reshape((num_labels, hid_size + 1))

    # パラメータの偏微分
    Theta1_grad = np.zeros(Theta1.shape)
    Theta2_grad = np.zeros(Theta2.shape)

    # 訓練データ数
    m = X.shape[0]

    # 訓練データの1列目にバイアス項に対応する1を追加
    X = np.hstack((np.ones((m, 1)), X))

    # 教師ラベルを1-of-K表記に変換
    lb = LabelBinarizer()
    lb.fit(y)
    y = lb.transform(y)

    J = 0
    for i in range(m):
        xi = X[i, :]
        yi = y[i]
        # forward propagation
        a1 = xi
        z2 = np.dot(Theta1, a1)
        a2 = sigmoid(z2)
        a2 = np.hstack((1, a2))
        z3 = np.dot(Theta2, a2)
        a3 = sigmoid(z3)
        J += sum(-yi * safe_log(a3) - (1 - yi) * safe_log(1 - a3))
        # backpropagation
        delta3 = a3 - yi
        delta2 = np.dot(Theta2.T, delta3) * sigmoidGradient(np.hstack((1, z2)))
        delta2 = delta2[1:]  # バイアス項に対応する要素を除外
        # ベクトル x ベクトル = 行列の演算をしなければならないので
        # 縦ベクトルへのreshapeが必要
        # 行数に-1を指定すると自動的に入る
        delta2 = delta2.reshape((-1, 1))
        delta3 = delta3.reshape((-1, 1))
        a1 = a1.reshape((-1, 1))
        a2 = a2.reshape((-1, 1))
        # 正則化ありのときのデルタの演算
        Theta1_grad += np.dot(delta2, a1.T)
        Theta2_grad += np.dot(delta3, a2.T)
    J /= m

    # 正則化項
    temp = 0.0;
    for j in range(hid_size):
        for k in range(1, in_size + 1):  # バイアスに対応する重みは加えない
            temp += Theta1[j, k] ** 2
    for j in range(num_labels):
        for k in range(1, hid_size + 1): # バイアスに対応する重みは加えない
            temp += Theta2[j, k] ** 2
    J += lam / (2.0 * m) * temp;

    # 偏微分の正則化項
    Theta1_grad /= m
    Theta1_grad[:, 1:] += (lam / m) * Theta1_grad[:, 1:]
    Theta2_grad /= m
    Theta2_grad[:, 1:] += (lam / m) * Theta2_grad[:, 1:]

    # ベクトルに変換
    grad = np.hstack((np.ravel(Theta1_grad), np.ravel(Theta2_grad)))

    print J
    return J, grad

def predict(Theta1, Theta2, X):
    m = X.shape[0]
    num_labels = Theta2.shape[0]
    # forward propagation
    X = np.hstack((np.ones((m, 1)), X))
    h1 = sigmoid(np.dot(X, Theta1.T))

    h1 = np.hstack((np.ones((m, 1)), h1))
    h2 = sigmoid(np.dot(h1, Theta2.T))

    return np.argmax(h2, axis=1)

if __name__ == "__main__":
    # 入力層のユニット数
    in_size = 28 * 28

    # 隠れ層のユニット数
    hid_size = 25

    # 出力層のユニット数（＝ラベル数）
    num_labels = 10

    # 訓練データをロード
    digits = fetch_mldata('MNIST original', data_home=".")
    X = digits.data
    X = X.astype(np.float64)
    X /= X.max()
    y = digits.target

    # ランダムに5000サンプルを選択
    p = np.random.random_integers(0, len(X) - 1, 5000)
    X_train = X[p, :]
    y_train = y[p]

    # データを可視化
    displayData(X_train)

    # パラメータをランダムに初期化
    initial_Theta1 = randInitializeWeights(in_size, hid_size)
    initial_Theta2 = randInitializeWeights(hid_size, num_labels)

    # パラメータをベクトルにフラット化
    initial_nn_params = np.hstack((np.ravel(initial_Theta1), np.ravel(initial_Theta2)))
    print initial_nn_params.shape

    # 正則化係数
    lam = 1.0

    # 初期状態のコストを計算
    J, grad = nnCostFunction(initial_nn_params, in_size, hid_size, num_labels, X_train, y_train, lam)
    print "initial cost:", J

    # Conjugate Gradientでパラメータ推定
    # NNはコスト関数と偏微分の計算が重複するため同じ関数（nnCostFunction）にまとめている
    # この場合、fmin_cgではなくminimizeを使用するとよい
    # minimize()はscipy 0.11.0以上が必要
    res = optimize.minimize(fun=nnCostFunction, x0=initial_nn_params, method="CG", jac=True,
                                  options={'maxiter':100, 'disp':True},
                                  args=(in_size, hid_size, num_labels, X_train, y_train, lam))
    nn_params = res.x

    # パラメータを分解
    Theta1 = nn_params[0:(in_size + 1) * hid_size].reshape((hid_size, in_size + 1))
    Theta2 = nn_params[(in_size + 1) * hid_size:].reshape((num_labels, hid_size + 1))

    # 隠れユニットを可視化
    displayData(Theta1[:, 1:])

    # 訓練データのラベルを予測して精度を求める
    pred = predict(Theta1, Theta2, X_train)
    print "*** training set accuracy"
    print confusion_matrix(y_train, pred)
    print classification_report(y_train, pred)

    # ランダムにテストデータを選んで精度を求める
    print "*** test set accuracy"
    p = np.random.random_integers(0, len(X), 2000)
    X_test = X[p, :]
    y_test = y[p]
    pred = predict(Theta1, Theta2, X_test)
    print confusion_matrix(y_test, pred)
    print classification_report(y_test, pred)
<commit_msg>訓練データとテストデータがかぶらないように修正<commit_after>#coding: utf-8
import numpy as np
from scipy import optimize
from matplotlib import pyplot
from sklearn.datasets import load_digits, fetch_mldata
from sklearn.preprocessing import LabelBinarizer
from sklearn.metrics import confusion_matrix, classification_report

"""
正則化多層パーセプトロンによる手書き文字認識
・バッチ処理
・共役勾配法（Conjugate Gradient Method）によるパラメータ最適化
"""

def displayData(X):
    """
    データからランダムに100サンプル選んで可視化
    画像データは8x8ピクセルを仮定
    """
    # ランダムに100サンプル選ぶ
    sel = np.random.permutation(X.shape[0])
    sel = sel[:100]
    X = X[sel, :]
    for index, data in enumerate(X):
        pyplot.subplot(10, 10, index + 1)
        pyplot.axis('off')
        image = data.reshape((28, 28))
        pyplot.imshow(image, cmap=pyplot.cm.gray_r,
                     interpolation='nearest')
    pyplot.show()

def randInitializeWeights(L_in, L_out):
    """
    (-epsilon_init, +epsilon_init) の範囲で
    重みをランダムに初期化した重み行列を返す
    """
    # 入力となる層にはバイアス項が入るので+1が必要なので注意
    epsilon_init = 0.12
    W = np.random.rand(L_out, 1 + L_in) * 2 * epsilon_init - epsilon_init
    return W

def sigmoid(z):
    return 1.0 / (1 + np.exp(-z))

def sigmoidGradient(z):
    return sigmoid(z) * (1 - sigmoid(z))

def safe_log(x, minval=0.0000000001):
    return np.log(x.clip(min=minval))

def nnCostFunction(nn_params, *args):
    """NNのコスト関数とその偏微分を求める"""
    in_size, hid_size, num_labels, X, y, lam = args

    # ニューラルネットの全パラメータを行列形式に復元
    Theta1 = nn_params[0:(in_size + 1) * hid_size].reshape((hid_size, in_size + 1))
    Theta2 = nn_params[(in_size + 1) * hid_size:].reshape((num_labels, hid_size + 1))

    # パラメータの偏微分
    Theta1_grad = np.zeros(Theta1.shape)
    Theta2_grad = np.zeros(Theta2.shape)

    # 訓練データ数
    m = X.shape[0]

    # 訓練データの1列目にバイアス項に対応する1を追加
    X = np.hstack((np.ones((m, 1)), X))

    # 教師ラベルを1-of-K表記に変換
    lb = LabelBinarizer()
    lb.fit(y)
    y = lb.transform(y)

    J = 0
    for i in range(m):
        xi = X[i, :]
        yi = y[i]
        # forward propagation
        a1 = xi
        z2 = np.dot(Theta1, a1)
        a2 = sigmoid(z2)
        a2 = np.hstack((1, a2))
        z3 = np.dot(Theta2, a2)
        a3 = sigmoid(z3)
        J += sum(-yi * safe_log(a3) - (1 - yi) * safe_log(1 - a3))
        # backpropagation
        delta3 = a3 - yi
        delta2 = np.dot(Theta2.T, delta3) * sigmoidGradient(np.hstack((1, z2)))
        delta2 = delta2[1:]  # バイアス項に対応する要素を除外
        # ベクトル x ベクトル = 行列の演算をしなければならないので
        # 縦ベクトルへのreshapeが必要
        # 行数に-1を指定すると自動的に入る
        delta2 = delta2.reshape((-1, 1))
        delta3 = delta3.reshape((-1, 1))
        a1 = a1.reshape((-1, 1))
        a2 = a2.reshape((-1, 1))
        # 正則化ありのときのデルタの演算
        Theta1_grad += np.dot(delta2, a1.T)
        Theta2_grad += np.dot(delta3, a2.T)
    J /= m

    # 正則化項
    temp = 0.0;
    for j in range(hid_size):
        for k in range(1, in_size + 1):  # バイアスに対応する重みは加えない
            temp += Theta1[j, k] ** 2
    for j in range(num_labels):
        for k in range(1, hid_size + 1): # バイアスに対応する重みは加えない
            temp += Theta2[j, k] ** 2
    J += lam / (2.0 * m) * temp;

    # 偏微分の正則化項
    Theta1_grad /= m
    Theta1_grad[:, 1:] += (lam / m) * Theta1_grad[:, 1:]
    Theta2_grad /= m
    Theta2_grad[:, 1:] += (lam / m) * Theta2_grad[:, 1:]

    # ベクトルに変換
    grad = np.hstack((np.ravel(Theta1_grad), np.ravel(Theta2_grad)))

    print "J =", J
    return J, grad

def predict(Theta1, Theta2, X):
    m = X.shape[0]
    num_labels = Theta2.shape[0]
    # forward propagation
    X = np.hstack((np.ones((m, 1)), X))
    h1 = sigmoid(np.dot(X, Theta1.T))

    h1 = np.hstack((np.ones((m, 1)), h1))
    h2 = sigmoid(np.dot(h1, Theta2.T))

    return np.argmax(h2, axis=1)

if __name__ == "__main__":
    # 入力層のユニット数
    in_size = 28 * 28

    # 隠れ層のユニット数
    hid_size = 25

    # 出力層のユニット数（＝ラベル数）
    num_labels = 10

    # 訓練データをロード
    digits = fetch_mldata('MNIST original', data_home=".")
    X = digits.data
    X = X.astype(np.float64)
    X /= X.max()
    y = digits.target

    # データをシャッフル
    p = np.random.permutation(len(X))
    X = X[p, :]
    y = y[p]

    # 最初の5000サンプルを選択
    X_train = X[:5000, :]
    y_train = y[:5000]

    # データを可視化
    displayData(X_train)

    # パラメータをランダムに初期化
    initial_Theta1 = randInitializeWeights(in_size, hid_size)
    initial_Theta2 = randInitializeWeights(hid_size, num_labels)

    # パラメータをベクトルにフラット化
    initial_nn_params = np.hstack((np.ravel(initial_Theta1), np.ravel(initial_Theta2)))

    # 正則化係数
    lam = 1.0

    # 初期状態のコストを計算
    J, grad = nnCostFunction(initial_nn_params, in_size, hid_size, num_labels, X_train, y_train, lam)

    # Conjugate Gradientでパラメータ推定
    # NNはコスト関数と偏微分の計算が重複するため同じ関数（nnCostFunction）にまとめている
    # この場合、fmin_cgではなくminimizeを使用するとよい
    # minimize()はscipy 0.11.0以上が必要
    res = optimize.minimize(fun=nnCostFunction, x0=initial_nn_params, method="CG", jac=True,
                                  options={'maxiter':70, 'disp':True},
                                  args=(in_size, hid_size, num_labels, X_train, y_train, lam))
    nn_params = res.x

    # パラメータを分解
    Theta1 = nn_params[0:(in_size + 1) * hid_size].reshape((hid_size, in_size + 1))
    Theta2 = nn_params[(in_size + 1) * hid_size:].reshape((num_labels, hid_size + 1))

    # 隠れユニットを可視化
    displayData(Theta1[:, 1:])

    # 訓練データのラベルを予測して精度を求める
    pred = predict(Theta1, Theta2, X_train)
    print "*** training set accuracy"
    print confusion_matrix(y_train, pred)
    print classification_report(y_train, pred)

    # 訓練データとかぶらない範囲からテストデータを選んで精度を求める
    print "*** test set accuracy"
    X_test = X[10000:12000, :]
    y_test = y[10000:12000]
    pred = predict(Theta1, Theta2, X_test)
    print len(pred)
    print confusion_matrix(y_test, pred)
    print classification_report(y_test, pred)
<|endoftext|>


<commit_before># flake8: NOQA
# "flake8: NOQA" to suppress warning "H104  File contains nothing but comments"

# class s_(object):

import numpy
import six

import cupy
from cupy import core
from cupy.creation import from_data
from cupy.manipulation import join

class AxisConcatenator(object):
    """Translates slice objects to concatenation along an axis.

    For detailed documentation on usage, see :func:`cupy.r_`.
    This implementation is partially borrowed from NumPy's one.

    """

    def _output_obj(self, obj, ndim, ndmin, trans1d):
        k2 = ndmin - ndim
        if trans1d < 0:
            trans1d += k2 + 1
        defaxes = list(six.moves.range(ndmin))
        k1 = trans1d
        axes = defaxes[:k1] + defaxes[k2:] + \
            defaxes[k1:k2]
        return obj.transpose(axes)

    def __init__(self, axis=0, matrix=False, ndmin=1, trans1d=-1):
        self.axis = axis
        self.trans1d = trans1d
        self.matrix = matrix
        self.ndmin = ndmin

    def __getitem__(self, key):
        trans1d = self.trans1d
        ndmin = self.ndmin
        objs = []
        scalars = []
        arraytypes = []
        scalartypes = []
        if isinstance(key, six.string_types):
            raise NotImplementedError
        if not isinstance(key, tuple):
            key = (key,)

        for i, k in enumerate(key):
            scalar = False
            if isinstance(k, slice):
                raise NotImplementedError
            elif isinstance(k, six.string_types):
                if i != 0:
                    raise ValueError(
                        'special directives must be the first entry.')
                raise NotImplementedError
            elif type(k) in numpy.ScalarType:
                newobj = from_data.array(k, ndmin=ndmin)
                scalars.append(i)
                scalar = True
                scalartypes.append(newobj.dtype)
            else:
                newobj = from_data.array(k, copy=False, ndmin=ndmin)
                if ndmin > 1:
                    ndim = from_data.array(k, copy=False).ndim
                    if trans1d != -1 and ndim < ndmin:
                        newobj = self._output_obj(newobj, ndim, ndmin, trans1d)

            objs.append(newobj)
            if not scalar and isinstance(newobj, core.ndarray):
                arraytypes.append(newobj.dtype)

        final_dtype = numpy.find_common_type(arraytypes, scalartypes)
        if final_dtype is not None:
            for k in scalars:
                objs[k] = objs[k].astype(final_dtype)

        return join.concatenate(tuple(objs), axis=self.axis)

    def __len__(self):
        return 0

class CClass(AxisConcatenator):

    def __init__(self):
        super(CClass, self).__init__(-1, ndmin=2, trans1d=0)

c_ = CClass()
"""Translates slice objects to concatenation along the second axis.

This is a CuPy object that corresponds to :func:`cupy.r_`, which is
useful because of its common occurrence. In particular, arrays will be
stacked along their last axis after being upgraded to at least 2-D with
1's post-pended to the shape (column vectors made out of 1-D arrays).

For detailed documentation, see :func:`r_`.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.c_`

Examples
--------
>>> a = cupy.array([[1, 2, 3]], dtype=np.int32)
>>> b = cupy.array([[4, 5, 6]], dtype=np.int32)
>>> cupy.c_[a, 0, 0, b]
array([[1, 2, 3, 0, 0, 4, 5, 6]], dtype=int32)

"""

class RClass(AxisConcatenator):

    def __init__(self):
        super(RClass, self).__init__()

r_ = RClass()
"""Translates slice objects to concatenation along the first axis.

This is a simple way to build up arrays quickly.
If the index expression contains comma separated arrays, then stack
them along their first axis.

This object can build up from normal CuPy arrays.
Therefore, the other objects (e.g. writing strings like '2,3,4',
or using imaginary numbers like [1,2,3j],
or using string integers like '-1') are not implemented yet
compared with NumPy.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.r_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.r_[a, 0, 0, b]
array([1, 2, 3, 0, 0, 4, 5, 6], dtype=int32)

"""

def indices(dimensions, dtype=int):
    """Return an array representing the indices of a grid.

    Compute an array where the subarrays contain index values 0,1,...
    varying only along the corresponding axis.

    Args:
        dimensions: The shape of the grid.
        dtype: Data type specifier. It is int by default.

    Returns:
        ndarray:
        The array of grid indices,
        ``grid.shape = (len(dimensions),) + tuple(dimensions)``.

    Examples
    --------
    >>> grid = cupy.indices((2, 3))
    >>> grid.shape
    (2, 2, 3)
    >>> grid[0]        # row indices
    array([[0, 0, 0],
           [1, 1, 1]])
    >>> grid[1]        # column indices
    array([[0, 1, 2],
           [0, 1, 2]])

    .. seealso:: :func:`numpy.indices`

    """
    dimensions = tuple(dimensions)
    N = len(dimensions)
    shape = (1,) * N
    res = cupy.empty((N,) + dimensions, dtype=dtype)
    for i, dim in enumerate(dimensions):
        res[i] = cupy.arange(dim, dtype=dtype).reshape(
            shape[:i] + (dim,) + shape[i + 1:]
        )
    return res

def ix_(*args):
    """Construct an open mesh from multiple sequences.

    This function takes N 1-D sequences and returns N outputs with N
    dimensions each, such that the shape is 1 in all but one dimension
    and the dimension with the non-unit shape value cycles through all
    N dimensions.

    Using `ix_` one can quickly construct index arrays that will index
    the cross product. ``a[cupy.ix_([1,3],[2,5])]`` returns the array
    ``[[a[1,2] a[1,5]], [a[3,2] a[3,5]]]``.

    Args:
        *args: 1-D sequences

    Returns:
        tuple of ndarrays:
        N arrays with N dimensions each, with N the number of input sequences.
        Together these arrays form an open mesh.

    Examples
    --------
    >>> a = cupy.arange(10).reshape(2, 5)
    >>> a
    array([[0, 1, 2, 3, 4],
           [5, 6, 7, 8, 9]])
    >>> ixgrid = cupy.ix_([0,1], [2,4])
    >>> ixgrid
    (array([[0],
           [1]]), array([[2, 4]]))

     .. seealso:: :func:`numpy.ix_`

    """
    out = []
    nd = len(args)
    for k, new in enumerate(args):
        new = from_data.asarray(new)
        if new.ndim != 1:
            raise ValueError("Cross index must be 1 dimensional")
        if new.size == 0:
            # Explicitly type empty arrays to avoid float default
            new = new.astype(numpy.intp)
        if cupy.issubdtype(new.dtype, cupy.bool_):
            new, = new.nonzero()
        new = new.reshape((1,) * k + (new.size,) + (1,) * (nd - k - 1))
        out.append(new)
    return tuple(out)

# TODO(okuta): Implement ravel_multi_index

# TODO(okuta): Implement unravel_index

# TODO(okuta): Implement diag_indices

# TODO(okuta): Implement diag_indices_from

# TODO(okuta): Implement mask_indices

# TODO(okuta): Implement tril_indices

# TODO(okuta): Implement tril_indices_from

# TODO(okuta): Implement triu_indices

# TODO(okuta): Implement triu_indices_from
<commit_msg>add unravel_index<commit_after># flake8: NOQA
# "flake8: NOQA" to suppress warning "H104  File contains nothing but comments"

# class s_(object):

import numpy
import six

import cupy
from cupy import core
from cupy.creation import from_data
from cupy.manipulation import join

class AxisConcatenator(object):
    """Translates slice objects to concatenation along an axis.

    For detailed documentation on usage, see :func:`cupy.r_`.
    This implementation is partially borrowed from NumPy's one.

    """

    def _output_obj(self, obj, ndim, ndmin, trans1d):
        k2 = ndmin - ndim
        if trans1d < 0:
            trans1d += k2 + 1
        defaxes = list(six.moves.range(ndmin))
        k1 = trans1d
        axes = defaxes[:k1] + defaxes[k2:] + \
            defaxes[k1:k2]
        return obj.transpose(axes)

    def __init__(self, axis=0, matrix=False, ndmin=1, trans1d=-1):
        self.axis = axis
        self.trans1d = trans1d
        self.matrix = matrix
        self.ndmin = ndmin

    def __getitem__(self, key):
        trans1d = self.trans1d
        ndmin = self.ndmin
        objs = []
        scalars = []
        arraytypes = []
        scalartypes = []
        if isinstance(key, six.string_types):
            raise NotImplementedError
        if not isinstance(key, tuple):
            key = (key,)

        for i, k in enumerate(key):
            scalar = False
            if isinstance(k, slice):
                raise NotImplementedError
            elif isinstance(k, six.string_types):
                if i != 0:
                    raise ValueError(
                        'special directives must be the first entry.')
                raise NotImplementedError
            elif type(k) in numpy.ScalarType:
                newobj = from_data.array(k, ndmin=ndmin)
                scalars.append(i)
                scalar = True
                scalartypes.append(newobj.dtype)
            else:
                newobj = from_data.array(k, copy=False, ndmin=ndmin)
                if ndmin > 1:
                    ndim = from_data.array(k, copy=False).ndim
                    if trans1d != -1 and ndim < ndmin:
                        newobj = self._output_obj(newobj, ndim, ndmin, trans1d)

            objs.append(newobj)
            if not scalar and isinstance(newobj, core.ndarray):
                arraytypes.append(newobj.dtype)

        final_dtype = numpy.find_common_type(arraytypes, scalartypes)
        if final_dtype is not None:
            for k in scalars:
                objs[k] = objs[k].astype(final_dtype)

        return join.concatenate(tuple(objs), axis=self.axis)

    def __len__(self):
        return 0

class CClass(AxisConcatenator):

    def __init__(self):
        super(CClass, self).__init__(-1, ndmin=2, trans1d=0)

c_ = CClass()
"""Translates slice objects to concatenation along the second axis.

This is a CuPy object that corresponds to :func:`cupy.r_`, which is
useful because of its common occurrence. In particular, arrays will be
stacked along their last axis after being upgraded to at least 2-D with
1's post-pended to the shape (column vectors made out of 1-D arrays).

For detailed documentation, see :func:`r_`.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.c_`

Examples
--------
>>> a = cupy.array([[1, 2, 3]], dtype=np.int32)
>>> b = cupy.array([[4, 5, 6]], dtype=np.int32)
>>> cupy.c_[a, 0, 0, b]
array([[1, 2, 3, 0, 0, 4, 5, 6]], dtype=int32)

"""

class RClass(AxisConcatenator):

    def __init__(self):
        super(RClass, self).__init__()

r_ = RClass()
"""Translates slice objects to concatenation along the first axis.

This is a simple way to build up arrays quickly.
If the index expression contains comma separated arrays, then stack
them along their first axis.

This object can build up from normal CuPy arrays.
Therefore, the other objects (e.g. writing strings like '2,3,4',
or using imaginary numbers like [1,2,3j],
or using string integers like '-1') are not implemented yet
compared with NumPy.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.r_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.r_[a, 0, 0, b]
array([1, 2, 3, 0, 0, 4, 5, 6], dtype=int32)

"""

def indices(dimensions, dtype=int):
    """Return an array representing the indices of a grid.

    Compute an array where the subarrays contain index values 0,1,...
    varying only along the corresponding axis.

    Args:
        dimensions: The shape of the grid.
        dtype: Data type specifier. It is int by default.

    Returns:
        ndarray:
        The array of grid indices,
        ``grid.shape = (len(dimensions),) + tuple(dimensions)``.

    Examples
    --------
    >>> grid = cupy.indices((2, 3))
    >>> grid.shape
    (2, 2, 3)
    >>> grid[0]        # row indices
    array([[0, 0, 0],
           [1, 1, 1]])
    >>> grid[1]        # column indices
    array([[0, 1, 2],
           [0, 1, 2]])

    .. seealso:: :func:`numpy.indices`

    """
    dimensions = tuple(dimensions)
    N = len(dimensions)
    shape = (1,) * N
    res = cupy.empty((N,) + dimensions, dtype=dtype)
    for i, dim in enumerate(dimensions):
        res[i] = cupy.arange(dim, dtype=dtype).reshape(
            shape[:i] + (dim,) + shape[i + 1:]
        )
    return res

def ix_(*args):
    """Construct an open mesh from multiple sequences.

    This function takes N 1-D sequences and returns N outputs with N
    dimensions each, such that the shape is 1 in all but one dimension
    and the dimension with the non-unit shape value cycles through all
    N dimensions.

    Using `ix_` one can quickly construct index arrays that will index
    the cross product. ``a[cupy.ix_([1,3],[2,5])]`` returns the array
    ``[[a[1,2] a[1,5]], [a[3,2] a[3,5]]]``.

    Args:
        *args: 1-D sequences

    Returns:
        tuple of ndarrays:
        N arrays with N dimensions each, with N the number of input sequences.
        Together these arrays form an open mesh.

    Examples
    --------
    >>> a = cupy.arange(10).reshape(2, 5)
    >>> a
    array([[0, 1, 2, 3, 4],
           [5, 6, 7, 8, 9]])
    >>> ixgrid = cupy.ix_([0,1], [2,4])
    >>> ixgrid
    (array([[0],
           [1]]), array([[2, 4]]))

     .. seealso:: :func:`numpy.ix_`

    """
    out = []
    nd = len(args)
    for k, new in enumerate(args):
        new = from_data.asarray(new)
        if new.ndim != 1:
            raise ValueError("Cross index must be 1 dimensional")
        if new.size == 0:
            # Explicitly type empty arrays to avoid float default
            new = new.astype(numpy.intp)
        if cupy.issubdtype(new.dtype, cupy.bool_):
            new, = new.nonzero()
        new = new.reshape((1,) * k + (new.size,) + (1,) * (nd - k - 1))
        out.append(new)
    return tuple(out)

# TODO(okuta): Implement ravel_multi_index

def unravel_index(indices, dims, order='C'):
    if order == 'C':
        dims = reversed(dims)
    elif order == 'F':
        pass
    else:
        raise ValueError("only 'C' or 'F' order is permitted")

    if (indices < 0).any():
        raise ValueError("invalid entry in index array")

    unraveled_coords = list()
    for dim in dims:
        if indices < 0
        unraveled_coords.append(indices % dim)
        indices = indices // dim

    if (indices > 0).any():
        raise ValueError("invalid entry in index array")

    return tuple(unraveled_coords)

# TODO(okuta): Implement diag_indices

# TODO(okuta): Implement diag_indices_from

# TODO(okuta): Implement mask_indices

# TODO(okuta): Implement tril_indices

# TODO(okuta): Implement tril_indices_from

# TODO(okuta): Implement triu_indices

# TODO(okuta): Implement triu_indices_from
<|endoftext|>


<commit_before># flake8: NOQA
# "flake8: NOQA" to suppress warning "H104  File contains nothing but comments"

# class s_(object):

import numpy
import six

import cupy
from cupy import core
from cupy.creation import from_data
from cupy.manipulation import join

class AxisConcatenator(object):
    """Translates slice objects to concatenation along an axis.

    For detailed documentation on usage, see :func:`cupy.r_`.
    This implementation is partially borrowed from NumPy's one.

    """

    def _output_obj(self, obj, ndim, ndmin, trans1d):
        k2 = ndmin - ndim
        if trans1d < 0:
            trans1d += k2 + 1
        defaxes = list(six.moves.range(ndmin))
        k1 = trans1d
        axes = defaxes[:k1] + defaxes[k2:] + \
            defaxes[k1:k2]
        return obj.transpose(axes)

    def __init__(self, axis=0, matrix=False, ndmin=1, trans1d=-1):
        self.axis = axis
        self.trans1d = trans1d
        self.matrix = matrix
        self.ndmin = ndmin

    def __getitem__(self, key):
        trans1d = self.trans1d
        ndmin = self.ndmin
        objs = []
        scalars = []
        arraytypes = []
        scalartypes = []
        if isinstance(key, six.string_types):
            raise NotImplementedError
        if not isinstance(key, tuple):
            key = (key,)

        for i, k in enumerate(key):
            scalar = False
            if isinstance(k, slice):
                raise NotImplementedError
            elif isinstance(k, six.string_types):
                if i != 0:
                    raise ValueError(
                        'special directives must be the first entry.')
                raise NotImplementedError
            elif type(k) in numpy.ScalarType:
                newobj = from_data.array(k, ndmin=ndmin)
                scalars.append(i)
                scalar = True
                scalartypes.append(newobj.dtype)
            else:
                newobj = from_data.array(k, copy=False, ndmin=ndmin)
                if ndmin > 1:
                    ndim = from_data.array(k, copy=False).ndim
                    if trans1d != -1 and ndim < ndmin:
                        newobj = self._output_obj(newobj, ndim, ndmin, trans1d)

            objs.append(newobj)
            if not scalar and isinstance(newobj, core.ndarray):
                arraytypes.append(newobj.dtype)

        final_dtype = numpy.find_common_type(arraytypes, scalartypes)
        if final_dtype is not None:
            for k in scalars:
                objs[k] = objs[k].astype(final_dtype)

        return join.concatenate(tuple(objs), axis=self.axis)

    def __len__(self):
        return 0

class CClass(AxisConcatenator):

    def __init__(self):
        super(CClass, self).__init__(-1, ndmin=2, trans1d=0)

c_ = CClass()
"""Translates slice objects to concatenation along the second axis.

This is a CuPy object that corresponds to :func:`cupy.r_`, which is
useful because of its common occurrence. In particular, arrays will be
stacked along their last axis after being upgraded to at least 2-D with
1's post-pended to the shape (column vectors made out of 1-D arrays).

For detailed documentation, see :func:`r_`.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.c_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.c_[a, 0, 0, b]
array([[1, 2, 3, 0, 0, 4, 5, 6]], dtype=int32)

"""

class RClass(AxisConcatenator):

    def __init__(self):
        super(RClass, self).__init__()

r_ = RClass()
"""Translates slice objects to concatenation along the first axis.

This is a simple way to build up arrays quickly.
If the index expression contains comma separated arrays, then stack
them along their first axis.

This object can build up from normal CuPy arrays.
Therefore, the other objects (e.g. writing strings like '2,3,4',
or using imaginary numbers like [1,2,3j],
or using string integers like '-1') are not implemented yet
compared with NumPy.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.r_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.r_[a, 0, 0, b]
array([1, 2, 3, 0, 0, 4, 5, 6], dtype=int32)

"""

# TODO(okuta): Implement indices

def ix_(*args):
    """Construct an open mesh from multiple sequences.

    This function takes N 1-D sequences and returns N outputs with N
    dimensions each, such that the shape is 1 in all but one dimension
    and the dimension with the non-unit shape value cycles through all
    N dimensions.

    Using `ix_` one can quickly construct index arrays that will index
    the cross product. ``a[cupy.ix_([1,3],[2,5])]`` returns the array
    ``[[a[1,2] a[1,5]], [a[3,2] a[3,5]]]``.

    Args:
        *args: 1-D sequences

    Returns:
        tuple of ndarrays:
        N arrays with N dimensions each, with N the number of input sequences.
        Together these arrays form an open mesh.

    Examples
    --------
    >>> a = cupy.arange(10).reshape(2, 5)
    >>> a
    array([[0, 1, 2, 3, 4],
           [5, 6, 7, 8, 9]])
    >>> ixgrid = cupy.ix_([0,1], [2,4])
    >>> ixgrid
    (array([[0],
           [1]]), array([[2, 4]]))

     .. seealso:: :func:`numpy.ix_`

    """
    out = []
    nd = len(args)
    for k, new in enumerate(args):
        new = from_data.asarray(new)
        if new.ndim != 1:
            raise ValueError("Cross index must be 1 dimensional")
        if new.size == 0:
            # Explicitly type empty arrays to avoid float default
            new = new.astype(numpy.intp)
        if cupy.issubdtype(new.dtype, cupy.bool_):
            new, = new.nonzero()
        new = new.reshape((1,) * k + (new.size,) + (1,) * (nd - k - 1))
        out.append(new)
    return tuple(out)

# TODO(okuta): Implement ravel_multi_index

# TODO(okuta): Implement unravel_index

# TODO(okuta): Implement diag_indices

# TODO(okuta): Implement diag_indices_from

# TODO(okuta): Implement mask_indices

# TODO(okuta): Implement tril_indices

# TODO(okuta): Implement tril_indices_from

# TODO(okuta): Implement triu_indices

# TODO(okuta): Implement triu_indices_from
<commit_msg>Fix doctest<commit_after># flake8: NOQA
# "flake8: NOQA" to suppress warning "H104  File contains nothing but comments"

# class s_(object):

import numpy
import six

import cupy
from cupy import core
from cupy.creation import from_data
from cupy.manipulation import join

class AxisConcatenator(object):
    """Translates slice objects to concatenation along an axis.

    For detailed documentation on usage, see :func:`cupy.r_`.
    This implementation is partially borrowed from NumPy's one.

    """

    def _output_obj(self, obj, ndim, ndmin, trans1d):
        k2 = ndmin - ndim
        if trans1d < 0:
            trans1d += k2 + 1
        defaxes = list(six.moves.range(ndmin))
        k1 = trans1d
        axes = defaxes[:k1] + defaxes[k2:] + \
            defaxes[k1:k2]
        return obj.transpose(axes)

    def __init__(self, axis=0, matrix=False, ndmin=1, trans1d=-1):
        self.axis = axis
        self.trans1d = trans1d
        self.matrix = matrix
        self.ndmin = ndmin

    def __getitem__(self, key):
        trans1d = self.trans1d
        ndmin = self.ndmin
        objs = []
        scalars = []
        arraytypes = []
        scalartypes = []
        if isinstance(key, six.string_types):
            raise NotImplementedError
        if not isinstance(key, tuple):
            key = (key,)

        for i, k in enumerate(key):
            scalar = False
            if isinstance(k, slice):
                raise NotImplementedError
            elif isinstance(k, six.string_types):
                if i != 0:
                    raise ValueError(
                        'special directives must be the first entry.')
                raise NotImplementedError
            elif type(k) in numpy.ScalarType:
                newobj = from_data.array(k, ndmin=ndmin)
                scalars.append(i)
                scalar = True
                scalartypes.append(newobj.dtype)
            else:
                newobj = from_data.array(k, copy=False, ndmin=ndmin)
                if ndmin > 1:
                    ndim = from_data.array(k, copy=False).ndim
                    if trans1d != -1 and ndim < ndmin:
                        newobj = self._output_obj(newobj, ndim, ndmin, trans1d)

            objs.append(newobj)
            if not scalar and isinstance(newobj, core.ndarray):
                arraytypes.append(newobj.dtype)

        final_dtype = numpy.find_common_type(arraytypes, scalartypes)
        if final_dtype is not None:
            for k in scalars:
                objs[k] = objs[k].astype(final_dtype)

        return join.concatenate(tuple(objs), axis=self.axis)

    def __len__(self):
        return 0

class CClass(AxisConcatenator):

    def __init__(self):
        super(CClass, self).__init__(-1, ndmin=2, trans1d=0)

c_ = CClass()
"""Translates slice objects to concatenation along the second axis.

This is a CuPy object that corresponds to :func:`cupy.r_`, which is
useful because of its common occurrence. In particular, arrays will be
stacked along their last axis after being upgraded to at least 2-D with
1's post-pended to the shape (column vectors made out of 1-D arrays).

For detailed documentation, see :func:`r_`.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.c_`

Examples
--------
>>> a = cupy.array([[1, 2, 3]], dtype=np.int32)
>>> b = cupy.array([[4, 5, 6]], dtype=np.int32)
>>> cupy.c_[a, 0, 0, b]
array([[1, 2, 3, 0, 0, 4, 5, 6]], dtype=int32)

"""

class RClass(AxisConcatenator):

    def __init__(self):
        super(RClass, self).__init__()

r_ = RClass()
"""Translates slice objects to concatenation along the first axis.

This is a simple way to build up arrays quickly.
If the index expression contains comma separated arrays, then stack
them along their first axis.

This object can build up from normal CuPy arrays.
Therefore, the other objects (e.g. writing strings like '2,3,4',
or using imaginary numbers like [1,2,3j],
or using string integers like '-1') are not implemented yet
compared with NumPy.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.r_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.r_[a, 0, 0, b]
array([1, 2, 3, 0, 0, 4, 5, 6], dtype=int32)

"""

# TODO(okuta): Implement indices

def ix_(*args):
    """Construct an open mesh from multiple sequences.

    This function takes N 1-D sequences and returns N outputs with N
    dimensions each, such that the shape is 1 in all but one dimension
    and the dimension with the non-unit shape value cycles through all
    N dimensions.

    Using `ix_` one can quickly construct index arrays that will index
    the cross product. ``a[cupy.ix_([1,3],[2,5])]`` returns the array
    ``[[a[1,2] a[1,5]], [a[3,2] a[3,5]]]``.

    Args:
        *args: 1-D sequences

    Returns:
        tuple of ndarrays:
        N arrays with N dimensions each, with N the number of input sequences.
        Together these arrays form an open mesh.

    Examples
    --------
    >>> a = cupy.arange(10).reshape(2, 5)
    >>> a
    array([[0, 1, 2, 3, 4],
           [5, 6, 7, 8, 9]])
    >>> ixgrid = cupy.ix_([0,1], [2,4])
    >>> ixgrid
    (array([[0],
           [1]]), array([[2, 4]]))

     .. seealso:: :func:`numpy.ix_`

    """
    out = []
    nd = len(args)
    for k, new in enumerate(args):
        new = from_data.asarray(new)
        if new.ndim != 1:
            raise ValueError("Cross index must be 1 dimensional")
        if new.size == 0:
            # Explicitly type empty arrays to avoid float default
            new = new.astype(numpy.intp)
        if cupy.issubdtype(new.dtype, cupy.bool_):
            new, = new.nonzero()
        new = new.reshape((1,) * k + (new.size,) + (1,) * (nd - k - 1))
        out.append(new)
    return tuple(out)

# TODO(okuta): Implement ravel_multi_index

# TODO(okuta): Implement unravel_index

# TODO(okuta): Implement diag_indices

# TODO(okuta): Implement diag_indices_from

# TODO(okuta): Implement mask_indices

# TODO(okuta): Implement tril_indices

# TODO(okuta): Implement tril_indices_from

# TODO(okuta): Implement triu_indices

# TODO(okuta): Implement triu_indices_from
<|endoftext|>


<commit_before>"""
.. Copyright (c) 2015 Marshall Farrier
   license http://opensource.org/licenses/MIT

Observe equity (:mod:`optbot.db.observe`)
=========================================

.. currentmodule:: optbot.db.observe

Add equities passed from command line to `active` collection in mongodb.

Examples
--------
To add 'ge' and 'f' to `active` collection:

    python3 observe.py ge f
"""
import _constants
import _locconst

import logging
logger = logging.getLogger(_constants.LOGNAME)
handler = logging.FileHandler(_locconst.LOGFILE)
formatter = logging.Formatter(_constants.LOGFMT)
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)

from functools import partial

import sys

import conn

def insert(equities, client):
    _db = client[_constants.DB]
    _active = _db[_constants.ACTIVE]
    _values = [{'equity': _eq} for _eq in equities]
    for _val in _values:
        if _active.find_one(_val) is not None:
            logger.info("{} already present in {}.{}".format(_val, _constants.DB, _constants.ACTIVE))
        else:
            _active.insert_one(_val)
            logger.info("{} inserted into {}.{}".format(_val, _constants.DB, _constants.ACTIVE))

if __name__ == '__main__':
    conn.job(partial(insert, sys.argv[1:]), logger)
<commit_msg>Use stable version of pymongo<commit_after>"""
.. Copyright (c) 2015 Marshall Farrier
   license http://opensource.org/licenses/MIT

Observe equity (:mod:`optbot.db.observe`)
=========================================

.. currentmodule:: optbot.db.observe

Add equities passed from command line to `active` collection in mongodb.

Examples
--------
To add 'ge' and 'f' to `active` collection:

    python observe.py ge f
"""
import _constants
import _locconst

import logging
logger = logging.getLogger(_constants.LOGNAME)
handler = logging.FileHandler(_locconst.LOGFILE)
formatter = logging.Formatter(_constants.LOGFMT)
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)

from functools import partial
import sys

from pymongo.errors import BulkWriteError

import conn

def insert(equities, client):
    _db = client[_constants.DB]
    _active = _db[_constants.ACTIVE]
    _bulk = _active.initialize_unordered_bulk_op()
    _values = [{'equity': _eq} for _eq in equities]
    for _val in _values:
        if _active.find_one(_val) is not None:
            logger.info("{} already present in {}.{}".format(_val, _constants.DB, _constants.ACTIVE))
        else:
            _bulk.insert(_val)
            logger.info("{} queued for insert into {}.{}".format(_val, _constants.DB, _constants.ACTIVE))
    try:
        _bulk.execute()
    except BulkWriteError:
        logger.exception("Error writing to database")

if __name__ == '__main__':
    conn.job(partial(insert, sys.argv[1:]), logger)
<|endoftext|>


<commit_before>import math
import string
import re
import bisect
import cvmfs_api

import GeoIP

gi = GeoIP.open("/var/lib/cvmfs-server/geo/GeoLiteCity.dat", GeoIP.GEOIP_STANDARD)
gi6 = GeoIP.open("/var/lib/cvmfs-server/geo/GeoLiteCityv6.dat", GeoIP.GEOIP_STANDARD)

positive_expire_secs = 60*60  # 1 hour

# function came from http://www.johndcook.com/python_longitude_latitude.html
def distance_on_unit_sphere(lat1, long1, lat2, long2):

    if (lat1 == lat2) and (long1 == long2):
        return 0.0

    # Convert latitude and longitude to
    # spherical coordinates in radians.
    degrees_to_radians = math.pi/180.0

    # phi = 90 - latitude
    phi1 = (90.0 - lat1)*degrees_to_radians
    phi2 = (90.0 - lat2)*degrees_to_radians

    # theta = longitude
    theta1 = long1*degrees_to_radians
    theta2 = long2*degrees_to_radians

    # Compute spherical distance from spherical coordinates.

    # For two locations in spherical coordinates
    # (1, theta, phi) and (1, theta, phi)
    # cosine( arc length ) =
    #    sin phi sin phi' cos(theta-theta') + cos phi cos phi'
    # distance = rho * arc length

    cos = (math.sin(phi1)*math.sin(phi2)*math.cos(theta1 - theta2) +
           math.cos(phi1)*math.cos(phi2))
    arc = math.acos( cos )

    # Remember to multiply arc by the radius of the earth
    # in your favorite set of units to get length.
    return arc

# Pattern including all allowed characters in addresses.
# The geoip api functions will further validate, but for paranoia's sake
#   (because I'm not sure how good the functions' error checking is), make
#   sure the names are limited to valid hostname characters.
# Include ':' for IPv6 addresses.
addr_pattern = re.compile('^[0-9a-zA-Z.:-]*$')

# expected geo api URL:  /cvmfs/<repo_name>/api/v<version>/geo/<path_info>
#   <repo_name> is repository name
#   <version> is the api version number, typically "1.0"
#   <path_info> is <caching_string>/<serverlist>
#     <caching_string> can be anything to assist in ensuring that those
#       clients wanting the same answer get responses cached together;
#       typically the name of their shared proxy
#     <serverlist> is a comma-separated list of N server names
# response: a comma-separated list of numbers specifying the order of the N
#    given servers numbered 1 to N from geographically closest to furthest
#    away from the requester that initiated the connection (the requester
#    is typically the proxy)

def api(path_info, repo_name, version, start_response, environ):

    start = string.find(path_info, '/') + 1
    if (start == 0):
        return cvmfs_api.bad_request(start_response, 'no slash in geo path')

    servers = string.split(path_info[start:], ",")

    rem_addr = ''
    if 'HTTP_X_FORWARDED_FOR' in environ:
        forwarded_for = environ['HTTP_X_FORWARDED_FOR']
        start = string.rfind(forwarded_for, ' ') + 1
        if (start == 0):
            start = string.rfind(forwarded_for, ',') + 1
        rem_addr = forwarded_for[start:]
    else:
        if 'REMOTE_ADDR' in environ:
            rem_addr = environ['REMOTE_ADDR']

    if (len(rem_addr) < 256) and addr_pattern.search(rem_addr):
        if rem_addr.find(':') != -1:
            gir_rem = gi6.record_by_addr_v6(rem_addr)
        else:
            gir_rem = gi.record_by_addr(rem_addr)
    else:
        gir_rem = None

    if gir_rem is None:
        return cvmfs_api.bad_request(start_response, 'remote addr not found in database')

    idx = 1
    arcs = []
    indexes = []

    onegood = False
    for server in servers:
        if (len(server) < 256) and addr_pattern.search(server):
            # try IPv4 first since that DB is better and most servers
            #    today are dual stack if they have IPv6
            gir_server = gi.record_by_name(server)
            if gir_server is None:
                gir_server = gi6.record_by_name_v6(server)
        else:
            gir_server = None

        if gir_server is None:
            # put it on the end of the list
            arc = float("inf")
        else:
            onegood = True
            arc = distance_on_unit_sphere(gir_rem['latitude'],
                                          gir_rem['longitude'],
                                          gir_server['latitude'],
                                          gir_server['longitude'])

        i = bisect.bisect(arcs, arc)
        arcs[i:i] = [ arc ]
        indexes[i:i] = [ str(idx) ]
        idx += 1

    if not onegood:
        # return a bad request only if all the server names were bad
        return cvmfs_api.bad_request(start_response, 'no server addr found in database')

    response_body = string.join(indexes, ',') + '\n'

    status = '200 OK'
    response_headers = [('Content-Type', 'text/plain'),
                  ('Cache-Control', 'max-age=' + str(positive_expire_secs)),
                  ('Content-Length', str(len(response_body)))]
    start_response(status, response_headers)

    return [response_body]

<commit_msg>use passed-in proxy as priority source address<commit_after>import math
import string
import re
import bisect
import socket
import cvmfs_api

import GeoIP

gi = GeoIP.open("/var/lib/cvmfs-server/geo/GeoLiteCity.dat", GeoIP.GEOIP_STANDARD)
gi6 = GeoIP.open("/var/lib/cvmfs-server/geo/GeoLiteCityv6.dat", GeoIP.GEOIP_STANDARD)

positive_expire_secs = 60*60  # 1 hour

# function came from http://www.johndcook.com/python_longitude_latitude.html
def distance_on_unit_sphere(lat1, long1, lat2, long2):

    if (lat1 == lat2) and (long1 == long2):
        return 0.0

    # Convert latitude and longitude to
    # spherical coordinates in radians.
    degrees_to_radians = math.pi/180.0

    # phi = 90 - latitude
    phi1 = (90.0 - lat1)*degrees_to_radians
    phi2 = (90.0 - lat2)*degrees_to_radians

    # theta = longitude
    theta1 = long1*degrees_to_radians
    theta2 = long2*degrees_to_radians

    # Compute spherical distance from spherical coordinates.

    # For two locations in spherical coordinates
    # (1, theta, phi) and (1, theta, phi)
    # cosine( arc length ) =
    #    sin phi sin phi' cos(theta-theta') + cos phi cos phi'
    # distance = rho * arc length

    cos = (math.sin(phi1)*math.sin(phi2)*math.cos(theta1 - theta2) +
           math.cos(phi1)*math.cos(phi2))
    arc = math.acos( cos )

    # Remember to multiply arc by the radius of the earth
    # in your favorite set of units to get length.
    return arc

# look up geo info for IPv4 or IPv6 address
# will return None if the address does not exist in the DB
def addr_geoinfo(addr):
    if addr.find(':') != -1:
        return(gi6.record_by_addr_v6(addr))
    else:
        return(gi.record_by_addr(addr))

# Pattern including all allowed characters in addresses.
# The geoip api functions will further validate, but for paranoia's sake
#   (because I'm not sure how good the functions' error checking is), make
#   sure the names are limited to valid hostname characters.
# Include ':' for IPv6 addresses.
addr_pattern = re.compile('^[0-9a-zA-Z.:-]*$')

# expected geo api URL:  /cvmfs/<repo_name>/api/v<version>/geo/<path_info>
#   <repo_name> is repository name
#   <version> is the api version number, typically "1.0"
#   <path_info> is <caching_string>/<serverlist>
#     <caching_string> can be anything to assist in ensuring that those
#       clients wanting the same answer get responses cached together;
#       typically the name of their shared proxy.  If this resolves to
#       a valid IP address, attempt to use that address as the source
#       IP rather than the address seen by the web server. The reason for
#       that is so it isn't possible for someone to poison a cache by
#       using a name for someone else's proxy.
#     <serverlist> is a comma-separated list of N server names
# response: a comma-separated list of numbers specifying the order of the N
#    given servers numbered 1 to N from geographically closest to furthest
#    away from the requester that initiated the connection (the requester
#    is typically the proxy)

def api(path_info, repo_name, version, start_response, environ):

    slash = path_info.find('/')
    if (slash == -1):
        return cvmfs_api.bad_request(start_response, 'no slash in geo path')

    caching_string = path_info[0:slash]
    servers = string.split(path_info[slash+1:], ",")

    gir_rem = None
    rem_addr = ''
    if caching_string.find('.'):
        # might be a FQDN, use it if it resolves to an address
        try:
            ai = socket.getaddrinfo(caching_string,80,0,0,socket.IPPROTO_TCP)
            arec = ai[0]
            # prefer IPv4 because the database is more accurate
            for info in ai:
                if info[0] == socket.AF_INET:
                    arec = info
                    break
            rem_addr = arec[4][0]
        except:
            pass
        else:
            gir_rem = addr_geoinfo(rem_addr)

    if gir_rem is None:
        rem_addr = ''
        if 'HTTP_X_FORWARDED_FOR' in environ:
            forwarded_for = environ['HTTP_X_FORWARDED_FOR']
            start = string.rfind(forwarded_for, ' ') + 1
            if (start == 0):
                start = string.rfind(forwarded_for, ',') + 1
            rem_addr = forwarded_for[start:]
        else:
            if 'REMOTE_ADDR' in environ:
                rem_addr = environ['REMOTE_ADDR']

        if (len(rem_addr) < 256) and addr_pattern.search(rem_addr):
            gir_rem = addr_geoinfo(rem_addr)

    if gir_rem is None:
        return cvmfs_api.bad_request(start_response, 'remote addr not found in database')

    idx = 1
    arcs = []
    indexes = []

    onegood = False
    for server in servers:
        if (len(server) < 256) and addr_pattern.search(server):
            # try IPv4 first since that DB is better and most servers
            #    today are dual stack if they have IPv6
            gir_server = gi.record_by_name(server)
            if gir_server is None:
                gir_server = gi6.record_by_name_v6(server)
        else:
            gir_server = None

        if gir_server is None:
            # put it on the end of the list
            arc = float("inf")
        else:
            onegood = True
            arc = distance_on_unit_sphere(gir_rem['latitude'],
                                          gir_rem['longitude'],
                                          gir_server['latitude'],
                                          gir_server['longitude'])
            #print "distance between " + \
            #    rem_addr + ' (' + str(gir_rem['latitude']) + ',' + str(gir_rem['longitude']) + ')' \
            #    + " and " + \
            #    server + ' (' + str(gir_server['latitude']) + ',' + str(gir_server['longitude']) + ')' + \
            #    " is " + str(arc)

        i = bisect.bisect(arcs, arc)
        arcs[i:i] = [ arc ]
        indexes[i:i] = [ str(idx) ]
        idx += 1

    if not onegood:
        # return a bad request only if all the server names were bad
        return cvmfs_api.bad_request(start_response, 'no server addr found in database')

    response_body = string.join(indexes, ',') + '\n'

    status = '200 OK'
    response_headers = [('Content-Type', 'text/plain'),
                  ('Cache-Control', 'max-age=' + str(positive_expire_secs)),
                  ('Content-Length', str(len(response_body)))]
    start_response(status, response_headers)

    return [response_body]

<|endoftext|>


<commit_before>"""Command to draw the revision tree of an RCS file."""

import os

from cvsgit.main import Command
from cvsgit.rcs import RCSFile
from cvsgit.i18n import _

class Rcsdump(Command):
    __doc__ = _(
    """Dump all changes on the HEAD branch of an RCS file.

    Usage: %prog [options] <rcsfile>

    Displays the changes on the HEAD branch of an RCS file for
    debugging purposes.
    """)

    def initialize_options(self):
        self.add_option('--checkout', metavar='REVISION', help=\
            _("Dump the content of the specified REVISION."))

    def finalize_options(self):
        if len(self.args) < 1:
            self.usage_error(_('missing RCS file argument'))
        elif len(self.args) == 1:
            self.rcsfile = self.args[0]
        else:
            self.usage_error(_('too many arguments'))

    def run(self):
        rcsfile = RCSFile(self.rcsfile)

        if self.options.checkout:
            return self.checkout(rcsfile, self.options.checkout)

        print 'Head: %s' % rcsfile.head
        print 'Branch: %s' % rcsfile.branch
        print 'Revision trail:',
        for revision in rcsfile.revisions():
            print revision,
        print ''
        for change in rcsfile.changes():
            rcsfile._print_revision(change.revision)

    def checkout(self, rcsfile, revision):
        size = os.stat(self.rcsfile).st_size
        print rcsfile.blob(revision, size_hint=size)
<commit_msg>Expand keywords in rcsdump<commit_after>"""Command to draw the revision tree of an RCS file."""

import os

from cvsgit.main import Command
from cvsgit.rcs import RCSFile
from cvsgit.i18n import _

# FIXME: RCS should expand keywords, not CVS
from cvsgit.cvs import CVS

class Rcsdump(Command):
    __doc__ = _(
    """Dump all changes on the HEAD branch of an RCS file.

    Usage: %prog [options] <rcsfile>

    Displays the changes on the HEAD branch of an RCS file for
    debugging purposes.
    """)

    def initialize_options(self):
        self.add_option('--checkout', metavar='REVISION', help=\
            _("Dump the content of the specified REVISION."))

    def finalize_options(self):
        if len(self.args) < 1:
            self.usage_error(_('missing RCS file argument'))
        elif len(self.args) == 1:
            self.rcsfile = self.args[0]
        else:
            self.usage_error(_('too many arguments'))

    def run(self):
        rcsfile = RCSFile(self.rcsfile)

        if self.options.checkout:
            return self.checkout(rcsfile, self.options.checkout)

        print 'Head: %s' % rcsfile.head
        print 'Branch: %s' % rcsfile.branch
        print 'Revision trail:',
        for revision in rcsfile.revisions():
            print revision,
        print ''
        for change in rcsfile.changes():
            rcsfile._print_revision(change.revision)

    def checkout(self, rcsfile, revision):
        change = rcsfile.change(revision)
        size = os.stat(self.rcsfile).st_size
        blob = rcsfile.blob(revision, size_hint=size)
        cvs = CVS(os.path.join(os.path.dirname(rcsfile.filename)), None)
        print cvs.expand_keywords(blob, change, rcsfile.filename, revision)
<|endoftext|>


<commit_before>"""
Authors: Tim Bedin, Ricardo Pascual, Tim Erwin

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

This module contains the ProcessUnit class.

"""

import os,logging

from cwsl.configuration import configuration
from cwsl.utils import utils
from cwsl.core.argument_creator import ArgumentCreator
from cwsl.core.file_creator import FileCreator
from cwsl.core.constraint import Constraint
from cwsl.core.scheduler import SimpleExecManager

module_logger = logging.getLogger('cwsl.core.process_unit')

class ProcessUnit(object):
    """ This class sets up the execution of an operation
    performed on a DataSet.

    This class takes in a list of input DataSets, an output
    pattern to write files to, the shell command that needs to be
    run and any output_constraints that need to be added to the
    output.

    It sets up the ArgumentCreator classes required to
    run a particular, single processing operation for all of
    the possible combinations in the input and output DataSets.

    This could be done using a VisTrails module directly, but doing it like
    this allows us to separate the GUI presentation from the execution.

    """

    def __init__(self,
                 inputlist,
                 output_pattern,
                 shell_command,
                 extra_constraints=set([]),
                 map_dict={},
                 cons_keywords={},
                 positional_args=[],
                 execution_options={}):

        """ The class takes in a DataSet object, constraints to change and
        the path to an executable. It has an self.execute() method that
        returns a FileCreator to be used as input to the next module.

        Extra constraints to be applied to the output are given by
        extra_constraints.

        map_dict is a dictionary linking constraint names in the input DataSets
        to new constraints in the output. e.g.
        if map_dict = {"model": "obs-model"} then the "model" constraint in the
        input is renamed to be the "obs-model" Constraint in the output.

        """

        if extra_constraints:
            module_logger.debug("extra_constraints are: {0}"
                                .format(extra_constraints))

        self.inputlist = inputlist
        self.cons_keywords = cons_keywords
        self.positional_args = positional_args
        self.execution_options = execution_options

        # The initial Constraints for the output are built from the
        # output file pattern.
        self.pattern_constraints = FileCreator.constraints_from_pattern(output_pattern)

        # Fill the empty output constraints from the input DataSets.
        fixed_constraints = self.fill_empty_constraints(extra_constraints)
        final_constraints = self.apply_mappings(map_dict, fixed_constraints)

        module_logger.debug("Final output constraints are: {0}".format(final_constraints))

        # Make a file_creator from the new, fixed constraints.
        self.file_creator = FileCreator(output_pattern, final_constraints)

        self.shell_command = shell_command

    def fill_empty_constraints(self, extra_constraints):

        out_cons_names = [cons.key for cons in extra_constraints]
        # These are constraints that exist in the pattern but not in
        # the extra_constraints - they must be filled from the input
        # if they exist there.
        unfilled_constraints = set([cons for cons in self.pattern_constraints
                                    if cons.key not in out_cons_names])

        empty_given_constraints = set([cons for cons in extra_constraints
                                       if not cons.values])

        missing_constraints = unfilled_constraints.union(empty_given_constraints)
        module_logger.debug("Constraints that need to be filled from the input are: {0}".
                            format(missing_constraints))

        module_logger.debug("Before filling, extra_constraints is: {0}"
                            .format(extra_constraints))

        # Any open constraints in the output should be filled by
        # the values from the input dataset.
        for empty in missing_constraints:
            found_constraints = []
            module_logger.debug("{0} is a missing constraint".format(empty))
            for input_ds in self.inputlist:
                found_constraints.append(input_ds.get_constraint(empty.key))

            filtered_founds = [cons for cons in found_constraints
                               if cons is not None]

            if len(filtered_founds) == 1:
                module_logger.debug("Adding Constraint {0}"
                                    .format(filtered_founds[0]))
                extra_constraints.add(filtered_founds[0])
            elif not filtered_founds:
                module_logger.debug("Replacing empty Constraint! {0}"
                                    .format(empty))
                extra_constraints.add(empty)
            else:
                names = [cons.key for cons in filtered_founds]
                all_vals = [cons.values for cons in filtered_founds]
                name = set(names)
                extra_vals = set()
                for vals in all_vals:
                    extra_vals.add(vals)
                new_con = Constraint(name, extra_vals)
                extra_constraints.add(new_con)
                module_logger.debug("(multiple filtered founds) Adding Constraint {0}"
                                    .format(new_con))

        module_logger.debug("Full output Constraints: {0}"
                            .format(extra_constraints))

        new_dict = {}
        for cons in extra_constraints:
            new_dict[cons.key] = set([])

        for constraint in extra_constraints:
            new_dict[constraint.key] = new_dict[constraint.key].union(constraint.values)

        fixed_constraints = set([Constraint(key, new_dict[key])
                                 for key in new_dict])
        return fixed_constraints

    def apply_mappings(self, map_dict, fixed_constraints):

        # Apply any "mappings" from the map_dict.
        module_logger.debug("map_dict for this ProcessUnit: {0}".format(map_dict))
        to_remove = []
        to_add = []

        for mapping in map_dict:
            in_name = mapping
            out_name = map_dict[mapping]
            module_logger.debug("Mapping info: inname: {0} outname {1}".format(in_name, out_name))

            module_logger.debug("searching for constraint for mapping: {0}".format(in_name))

            for cons in fixed_constraints:
                print cons.key, mapping
                if cons.key == out_name:
                    module_logger.debug("Replacing Constraint: {0}".format(cons))

                    # Look for the matching key in the inputs.
                    found_cons = [inDS.get_constraint(in_name) for inDS in self.inputlist]

                    to_remove.append(cons)
                    merged_cons = set(*[cons.values for cons in found_cons])
                    to_add.append(Constraint(out_name, merged_cons))

        module_logger.debug("to add: {0}".format(to_add))
        module_logger.debug("to remove: {0}".format(to_remove))

        for cons in to_remove:
            fixed_constraints.remove(cons)
        for cons in to_add:
            fixed_constraints.add(cons)

        return fixed_constraints

    def execute(self, simulate=False):
        """ This method runs the actual process."""

        # Check that cws_ctools_path is set
        if not configuration.cwsl_ctools_path or not os.path.exists(configuration.cwsl_ctools_path):
            raise Exception("cwsl_ctools_path is not set in package options")

        # We now create a looper to compare all the input Datasets with
        # the output fileCreators.
        this_looper = ArgumentCreator(self.inputlist, self.file_creator)
        module_logger.debug("Created ArgumentCreator: {0}".format(this_looper))

        #TODO determine scheduler from user options.
        scheduler = SimpleExecManager(noexec=simulate)
        if self.execution_options.has_key('required_modules'):
            scheduler.add_module_deps(self.execution_options['required_modules'])

        #Add environment variables to the script and the current environment.
        scheduler.add_environment_variables({'CWSL_CTOOLS':configuration.cwsl_ctools_path})
        os.environ['CWSL_CTOOLS'] = configuration.cwsl_ctools_path
        scheduler.add_python_paths([os.path.join(configuration.cwsl_ctools_path,'pythonlib')])

        # For every valid possible combination, apply any positional and
        # keyword args, then add the command to the scheduler.
        for combination in this_looper:
            module_logger.debug("Combination: " + str(combination))
            if combination:
                in_files, out_files = self.get_fullnames((combination[0], combination[1]))
                this_dict = combination[2]

                module_logger.info("in_files are:")
                module_logger.info(in_files)
                module_logger.info("out_files are:")
                module_logger.info(out_files)

                base_cmd_list = [self.shell_command] + in_files + out_files

                # Now apply any keyword arguments and positional args.
                keyword_command_list = self.apply_keyword_args(base_cmd_list, this_dict)
                final_command_list = self.apply_positional_args(keyword_command_list, this_dict)

                # Generate the annotation string.
                try:
                    annotation = utils.build_metadata(final_command_list)
                except NameError:
                    annotation = None

                # The subprocess / queue submission is done here.
                scheduler.add_cmd(final_command_list, out_files, annotation=annotation)

        scheduler.submit()

        # The scheduler is kept for testing purposes.
        self.scheduler = scheduler

        return self.file_creator

    def apply_keyword_args(self, command_list, kw_cons_dict, prefix='--'):
        """ Add keywords from the keyword constraint dictionary to the command list."""

        for keyword in self.cons_keywords:
            associated_cons_name = self.cons_keywords[keyword]
            this_att_value = kw_cons_dict[associated_cons_name]

            command_list.append(prefix + keyword + ' ' + this_att_value)

        return command_list

    def apply_positional_args(self, arg_list, constraint_dict):
        """ Add positional args to a list of command arguments. """

        for arg_tuple in self.positional_args:
            arg_name = arg_tuple[0]
            position = arg_tuple[1]

            try:
                if arg_tuple[2] == 'raw':
                    this_att_value = arg_name
            except IndexError:
                this_att_value = constraint_dict[arg_name]

            if position != -1:
                position += 1   # +1 because arg_list[0] is the actual command!
                arg_list.insert(position, this_att_value)
            else:
                arg_list.append(this_att_value)

        return arg_list

    def get_fullnames(self, combination):
        required_atts = ["path_dir", "filename"]
        in_files = []

        for qs in combination[0]:
            try:
                in_files += [infile.full_path
                             for infile in qs.only(*required_atts)]
            except AttributeError:
                in_files += [infile.full_path for infile in qs]

        out_files = []
        for qs in combination[1]:
            try:
                out_files += [outfile.full_path
                              for outfile in qs.only(*required_atts)]
            except AttributeError:
                out_files += [outfile.full_path for outfile in qs]

        return in_files, out_files

# Exception Classes
class EmptyOverwriteError(Exception):
    pass
<commit_msg>Initial implementation of empty constraint errors.<commit_after>"""
Authors: Tim Bedin, Ricardo Pascual, Tim Erwin

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

This module contains the ProcessUnit class.

"""

import os,logging

from cwsl.configuration import configuration
from cwsl.utils import utils
from cwsl.core.argument_creator import ArgumentCreator
from cwsl.core.file_creator import FileCreator
from cwsl.core.constraint import Constraint
from cwsl.core.scheduler import SimpleExecManager

module_logger = logging.getLogger('cwsl.core.process_unit')

class ProcessUnit(object):
    """ This class sets up the execution of an operation
    performed on a DataSet.

    This class takes in a list of input DataSets, an output
    pattern to write files to, the shell command that needs to be
    run and any output_constraints that need to be added to the
    output.

    It sets up the ArgumentCreator classes required to
    run a particular, single processing operation for all of
    the possible combinations in the input and output DataSets.

    This could be done using a VisTrails module directly, but doing it like
    this allows us to separate the GUI presentation from the execution.

    """

    def __init__(self,
                 inputlist,
                 output_pattern,
                 shell_command,
                 extra_constraints=set([]),
                 map_dict={},
                 cons_keywords={},
                 positional_args=[],
                 execution_options={}):

        """ The class takes in a DataSet object, constraints to change and
        the path to an executable. It has an self.execute() method that
        returns a FileCreator to be used as input to the next module.

        Extra constraints to be applied to the output are given by
        extra_constraints.

        map_dict is a dictionary linking constraint names in the input DataSets
        to new constraints in the output. e.g.
        if map_dict = {"model": "obs-model"} then the "model" constraint in the
        input is renamed to be the "obs-model" Constraint in the output.

        """

        if extra_constraints:
            module_logger.debug("extra_constraints are: {0}"
                                .format(extra_constraints))

        self.inputlist = inputlist
        self.cons_keywords = cons_keywords
        self.positional_args = positional_args
        self.execution_options = execution_options

        # The initial Constraints for the output are built from the
        # output file pattern.
        self.pattern_constraints = FileCreator.constraints_from_pattern(output_pattern)

        # Fill the empty output constraints from the input DataSets.
        fixed_constraints = self.fill_empty_constraints(extra_constraints)
        final_constraints = self.apply_mappings(map_dict, fixed_constraints)

        module_logger.debug("Final output constraints are: {0}".format(final_constraints))

        # Make a file_creator from the new, fixed constraints.
        self.file_creator = FileCreator(output_pattern, final_constraints)

        self.shell_command = shell_command

    def fill_empty_constraints(self, extra_constraints):

        out_cons_names = []
        for cons in extra_constraints:
            if not cons.values:
                raise EmptyOverwriteError("Constraint on {} is empty - can not overwrite existing constraint"
                                          .format(cons.key))
            else:
                out_cons_names.append(cons.key)

        # These are constraints that exist in the pattern but not in
        # the extra_constraints - they must be filled from the input
        # if they exist there.
        unfilled_constraints = set([cons for cons in self.pattern_constraints
                                    if cons.key not in out_cons_names])

        empty_given_constraints = set([cons for cons in extra_constraints
                                       if not cons.values])

        missing_constraints = unfilled_constraints.union(empty_given_constraints)
        module_logger.debug("Constraints that need to be filled from the input are: {0}".
                            format(missing_constraints))

        module_logger.debug("Before filling, extra_constraints is: {0}"
                            .format(extra_constraints))

        # Any open constraints in the output should be filled by
        # the values from the input dataset.
        for empty in missing_constraints:
            found_constraints = []
            module_logger.debug("{0} is a missing constraint".format(empty))
            for input_ds in self.inputlist:
                found_constraints.append(input_ds.get_constraint(empty.key))

            filtered_founds = [cons for cons in found_constraints
                               if cons is not None]

            if len(filtered_founds) == 1:
                module_logger.debug("Adding Constraint {0}"
                                    .format(filtered_founds[0]))
                extra_constraints.add(filtered_founds[0])
            elif not filtered_founds:
                module_logger.debug("Replacing empty Constraint! {0}"
                                    .format(empty))
                extra_constraints.add(empty)
            else:
                names = [cons.key for cons in filtered_founds]
                all_vals = [cons.values for cons in filtered_founds]
                name = set(names)
                extra_vals = set()
                for vals in all_vals:
                    extra_vals.add(vals)
                new_con = Constraint(name, extra_vals)
                extra_constraints.add(new_con)
                module_logger.debug("(multiple filtered founds) Adding Constraint {0}"
                                    .format(new_con))

        module_logger.debug("Full output Constraints: {0}"
                            .format(extra_constraints))

        new_dict = {}
        for cons in extra_constraints:
            new_dict[cons.key] = set([])

        for constraint in extra_constraints:
            new_dict[constraint.key] = new_dict[constraint.key].union(constraint.values)

        fixed_constraints = set([Constraint(key, new_dict[key])
                                 for key in new_dict])
        return fixed_constraints

    def apply_mappings(self, map_dict, fixed_constraints):

        # Apply any "mappings" from the map_dict.
        module_logger.debug("map_dict for this ProcessUnit: {0}".format(map_dict))
        to_remove = []
        to_add = []

        for mapping in map_dict:
            in_name = mapping
            out_name = map_dict[mapping]
            module_logger.debug("Mapping info: inname: {0} outname {1}".format(in_name, out_name))

            module_logger.debug("searching for constraint for mapping: {0}".format(in_name))

            for cons in fixed_constraints:
                print cons.key, mapping
                if cons.key == out_name:
                    module_logger.debug("Replacing Constraint: {0}".format(cons))

                    # Look for the matching key in the inputs.
                    found_cons = [inDS.get_constraint(in_name) for inDS in self.inputlist]

                    to_remove.append(cons)
                    merged_cons = set(*[cons.values for cons in found_cons])
                    to_add.append(Constraint(out_name, merged_cons))

        module_logger.debug("to add: {0}".format(to_add))
        module_logger.debug("to remove: {0}".format(to_remove))

        for cons in to_remove:
            fixed_constraints.remove(cons)
        for cons in to_add:
            fixed_constraints.add(cons)

        return fixed_constraints

    def execute(self, simulate=False):
        """ This method runs the actual process."""

        # Check that cws_ctools_path is set
        if not configuration.cwsl_ctools_path or not os.path.exists(configuration.cwsl_ctools_path):
            raise Exception("cwsl_ctools_path is not set in package options")

        # We now create a looper to compare all the input Datasets with
        # the output fileCreators.
        this_looper = ArgumentCreator(self.inputlist, self.file_creator)
        module_logger.debug("Created ArgumentCreator: {0}".format(this_looper))

        #TODO determine scheduler from user options.
        scheduler = SimpleExecManager(noexec=simulate)
        if self.execution_options.has_key('required_modules'):
            scheduler.add_module_deps(self.execution_options['required_modules'])

        #Add environment variables to the script and the current environment.
        scheduler.add_environment_variables({'CWSL_CTOOLS':configuration.cwsl_ctools_path})
        os.environ['CWSL_CTOOLS'] = configuration.cwsl_ctools_path
        scheduler.add_python_paths([os.path.join(configuration.cwsl_ctools_path,'pythonlib')])

        # For every valid possible combination, apply any positional and
        # keyword args, then add the command to the scheduler.
        for combination in this_looper:
            module_logger.debug("Combination: " + str(combination))
            if combination:
                in_files, out_files = self.get_fullnames((combination[0], combination[1]))
                this_dict = combination[2]

                module_logger.info("in_files are:")
                module_logger.info(in_files)
                module_logger.info("out_files are:")
                module_logger.info(out_files)

                base_cmd_list = [self.shell_command] + in_files + out_files

                # Now apply any keyword arguments and positional args.
                keyword_command_list = self.apply_keyword_args(base_cmd_list, this_dict)
                final_command_list = self.apply_positional_args(keyword_command_list, this_dict)

                # Generate the annotation string.
                try:
                    annotation = utils.build_metadata(final_command_list)
                except NameError:
                    annotation = None

                # The subprocess / queue submission is done here.
                scheduler.add_cmd(final_command_list, out_files, annotation=annotation)

        scheduler.submit()

        # The scheduler is kept for testing purposes.
        self.scheduler = scheduler

        return self.file_creator

    def apply_keyword_args(self, command_list, kw_cons_dict, prefix='--'):
        """ Add keywords from the keyword constraint dictionary to the command list."""

        for keyword in self.cons_keywords:
            associated_cons_name = self.cons_keywords[keyword]
            this_att_value = kw_cons_dict[associated_cons_name]

            command_list.append(prefix + keyword + ' ' + this_att_value)

        return command_list

    def apply_positional_args(self, arg_list, constraint_dict):
        """ Add positional args to a list of command arguments. """

        for arg_tuple in self.positional_args:
            arg_name = arg_tuple[0]
            position = arg_tuple[1]

            try:
                if arg_tuple[2] == 'raw':
                    this_att_value = arg_name
            except IndexError:
                this_att_value = constraint_dict[arg_name]

            if position != -1:
                position += 1   # +1 because arg_list[0] is the actual command!
                arg_list.insert(position, this_att_value)
            else:
                arg_list.append(this_att_value)

        return arg_list

    def get_fullnames(self, combination):
        required_atts = ["path_dir", "filename"]
        in_files = []

        for qs in combination[0]:
            try:
                in_files += [infile.full_path
                             for infile in qs.only(*required_atts)]
            except AttributeError:
                in_files += [infile.full_path for infile in qs]

        out_files = []
        for qs in combination[1]:
            try:
                out_files += [outfile.full_path
                              for outfile in qs.only(*required_atts)]
            except AttributeError:
                out_files += [outfile.full_path for outfile in qs]

        return in_files, out_files

# Exception Classes
class EmptyOverwriteError(Exception):
    pass
<|endoftext|>


<commit_before># Copyright (c) 2013, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import cybox.bindings.cybox_common as common_binding
from cybox.common import Contributor

class Personnel(object):
    def __init__(self):
        pass

    @classmethod
    def object_from_dict(cls, personnel_attributes):
        """Create the Personnel object representation from an input dictionary"""
        personnel_type = common_binding.PersonnelType()
        for contributor_dict in personnel_attributes:
            contributor_type = Contributor.object_from_dict(contributor_dict)
            if contributor_type.hasContent_(): personnel_type.add_Contributor(contributor_type)
        return personnel_type

    @classmethod
    def dict_from_object(cls, personnel_element):
        """Parse and return a dictionary for a Personnel object"""
        pass
<commit_msg>Updated Personnel for cybox 2.0 and current interfaces<commit_after># Copyright (c) 2013, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import cybox.bindings.cybox_common as common_binding
from cybox.common import Contributor

class Personnel(cybox.EntityList):
    _contained_type = Contributor
    _binding_class = common_binding.PersonnelType

    def __init__(self):
        super(Contributor, self).__init__()

    @staticmethod
    def _set_list(binding_obj, list_):
        binding_obj.set_Contributor(list_)

    @staticmethod
    def _get_list(binding_obj):
        return binding_obj.get_Contributor()<|endoftext|>