<commit_before>#!/usr/bin/env python

import sys
import os.path
import logging

import mapzen.whosonfirst.utils

# see notes in (local) whosonfirst.py about namespaces
# and anger (20150723/thisisaaronland)

import whosonfirst

if __name__ == '__main__':

    import optparse
    import ConfigParser

    opt_parser = optparse.OptionParser()

    opt_parser.add_option('-s', '--source', dest='source', action='store', default='None', help='')
    opt_parser.add_option('-c', '--config', dest='config', action='store', default='None', help='')

    # opt_parser.add_option('-d', '--database', dest='database', action='store', default='whosonfirst_pip', help='')
    # opt_parser.add_option('-u', '--username', dest='username', action='store', default='postgres', help='')
    # opt_parser.add_option('-p', '--password', dest='password', action='store', default=None, help='')

    opt_parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Be chatty (default is false)')
    options, args = opt_parser.parse_args()

    if options.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    if not os.path.exists(options.config):
        logging.error("Missing config file")
        sys.exit()

    cfg = ConfigParser.ConfigParser()
    cfg.read(options.config)

    db_host = cfg.get('whosonfirst', 'db_host')
    db_name = cfg.get('whosonfirst', 'db_name')
    db_user = cfg.get('whosonfirst', 'db_user')
    db_pswd = cfg.get('whosonfirst', 'db_pswd')

    dsn = "host=%s dbname=%s user=%s password=%s" % (db_host, db_name, db_user, db_pswd)
    db = whosonfirst.lookup(dsn)

    source = os.path.abspath(options.source)
    crawl = mapzen.whosonfirst.utils.crawl(source, inflate=True)

    for feature in crawl:
        logging.debug("import feature %s" % feature.get('id', "UNKNOWN"))

        geom = feature['geometry']

        if geom['type'] == 'Point':
            logging.warning("skipping because %s is a point" % id)
            continue

        db.import_feature(feature)

    sys.exit()

<commit_msg>remove old unused parse options<commit_after>#!/usr/bin/env python

import sys
import os.path
import logging

import mapzen.whosonfirst.utils

# see notes in (local) whosonfirst.py about namespaces
# and anger (20150723/thisisaaronland)

import whosonfirst

if __name__ == '__main__':

    import optparse
    import ConfigParser

    opt_parser = optparse.OptionParser()

    opt_parser.add_option('-s', '--source', dest='source', action='store', default='None', help='')
    opt_parser.add_option('-c', '--config', dest='config', action='store', default='None', help='')
    opt_parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Be chatty (default is false)')

    options, args = opt_parser.parse_args()

    if options.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    if not os.path.exists(options.config):
        logging.error("Missing config file")
        sys.exit()

    cfg = ConfigParser.ConfigParser()
    cfg.read(options.config)

    db_host = cfg.get('whosonfirst', 'db_host')
    db_name = cfg.get('whosonfirst', 'db_name')
    db_user = cfg.get('whosonfirst', 'db_user')
    db_pswd = cfg.get('whosonfirst', 'db_pswd')

    dsn = "host=%s dbname=%s user=%s password=%s" % (db_host, db_name, db_user, db_pswd)
    db = whosonfirst.lookup(dsn)

    source = os.path.abspath(options.source)
    crawl = mapzen.whosonfirst.utils.crawl(source, inflate=True)

    for feature in crawl:
        logging.debug("import feature %s" % feature.get('id', "UNKNOWN"))

        geom = feature['geometry']

        if geom['type'] == 'Point':
            logging.warning("skipping because %s is a point" % id)
            continue

        db.import_feature(feature)

    sys.exit()

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Construct a neural network model, support vector and decision trees regression models from the data
"""

import numpy as np
import pickle
import sklearn
from lasagne import nonlinearities
from lasagne.layers import DenseLayer
from lasagne.layers import InputLayer
from multiprocessing import Process
from nolearn.lasagne import NeuralNet
from sklearn.linear_model import RidgeCV, BayesianRidge, LassoCV
from sklearn.svm import SVR

__author__ = "Pearl Philip"
__credits__ = "David Beck"
__license__ = "BSD 3-Clause License"
__maintainer__ = "Pearl Philip"
__email__ = "pphilip@uw.edu"
__status__ = "Development"

NODES = 10
NN_PICKLE = '../trained_networks/nn_data.pkl'
SVM_PICKLE = '../trained_networks/svm_data.pkl'
DT_PICKLE = '../trained_networks/dt_data.pkl'
RR_PICKLE = '../trained_networks/rr_data.pkl'
BRR_PICKLE = '../trained_networks/brr_data.pkl'
LASSO_PICKLE = '../trained_networks/lasso_data.pkl'

def run_models(x_train, y_train, x_test, y_test):
    """
    Driving all machine learning models as parallel processes.
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """
    p1 = Process(target=build_nn, args=(x_train, y_train, x_test, y_test))
    p1.start()
    p2 = Process(target=build_svm, args=(x_train, y_train, x_test, y_test))
    p2.start()
    p3 = Process(target=build_tree, args=(x_train, y_train, x_test, y_test))
    p3.start()
    p4 = Process(target=build_ridge, args=(x_train, y_train, x_test, y_test))
    p4.start()
    p5 = Process(target=build_bayesian_rr, args=(x_train, y_train, x_test, y_test))
    p5.start()
    p6 = Process(target=build_lasso, args=(x_train, y_train, x_test, y_test))
    p6.start()

    p1.join()
    p2.join()
    p3.join()
    p4.join()
    p5.join()
    p6.join()

    return

def build_nn(x_train, y_train, x_test, y_test):
    """
    Constructing a regression neural network model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """

    # Create classification model
    net = NeuralNet(layers=[('input', InputLayer),
                            ('hidden0', DenseLayer),
                            ('hidden1', DenseLayer),
                            ('output', DenseLayer)],
                    input_shape=(None, x_train.shape[1]),
                    hidden0_num_units=NODES,
                    hidden0_nonlinearity=nonlinearities.softmax,
                    hidden1_num_units=NODES,
                    hidden1_nonlinearity=nonlinearities.softmax,
                    output_num_units=len(np.unique(y_train)),
                    output_nonlinearity=nonlinearities.softmax,
                    update_learning_rate=0.1,
                    verbose=1,
                    max_epochs=100)

    param_grid = {'hidden0_num_units': [1, 4, 17, 25],
                  'hidden0_nonlinearity':
                  [nonlinearities.sigmoid, nonlinearities.softmax],
                  'hidden1_num_units': [1, 4, 17, 25],
                  'hidden1_nonlinearity':
                  [nonlinearities.sigmoid, nonlinearities.softmax],
                  'update_learning_rate': [0.01, 0.1, 0.5]}

    # Finding the optimal set of params for each variable in the training of the neural network
    clf = sklearn.grid_search.GridSearchCV(net, param_grid, verbose=0, n_jobs=3, cv=3)
    clf.fit(x_train, y_train)

    y_pred = clf.predict(x_test)
    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)

    with open(NN_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(net, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_svm(x_train, y_train, x_test, y_test):
    """
    Constructing a support vector regression model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """

    clf = SVR()
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)

    with open(SVM_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_tree(x_train, y_train, x_test, y_test):
    """
    Constructing a decision trees regression model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """
    clf = sklearn.tree.DecisionTreeRegressor()
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)

    with open(DT_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_ridge(x_train, y_train, x_test, y_test):
    """
    Constructing a ridge regression model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """
    clf = RidgeCV(alphas=[0.01, 0.1, 1.0, 10.0])
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)
    # Optimal ridge regression alpha value from CV
    ridge_alpha = clf.alpha_

    with open(RR_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(ridge_alpha, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_bayesian_rr(x_train, y_train, x_test, y_test):
    """
    Constructing a Bayesian ridge regression model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """
    clf = BayesianRidge()
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)
    # Optimal ridge regression alpha value from CV
    ridge_alpha = clf.alpha_

    with open(BRR_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(ridge_alpha, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_lasso(x_train, y_train, x_test, y_test):
    """
    Constructing a Lasso linear model with cross validation from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """

    clf = LassoCV()
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)
    # Optimal ridge regression alpha value from CV
    lasso_alpha = clf.alpha_

    with open(LASSO_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(lasso_alpha, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return
<commit_msg>Removed multiprocessing module due to memory shortage, added five-fold cross-validation arg, corrected module import for GridSearchCV, updated neural net for regression instead of classification<commit_after>#!/usr/bin/env python

"""
Construct a neural network model, support vector and decision trees regression models from the data
"""

import numpy as np
import pickle
import sklearn
from lasagne import nonlinearities
from lasagne.layers import DenseLayer
from lasagne.layers import InputLayer
from nolearn.lasagne import NeuralNet
from sklearn.linear_model import RidgeCV, BayesianRidge, LassoCV
from sklearn.svm import SVR

__author__ = "Pearl Philip"
__credits__ = "David Beck"
__license__ = "BSD 3-Clause License"
__maintainer__ = "Pearl Philip"
__email__ = "pphilip@uw.edu"
__status__ = "Development"

NODES = 10
NN_PICKLE = '../trained_networks/nn_data.pkl'
SVM_PICKLE = '../trained_networks/svm_data.pkl'
DT_PICKLE = '../trained_networks/dt_data.pkl'
RR_PICKLE = '../trained_networks/rr_data.pkl'
BRR_PICKLE = '../trained_networks/brr_data.pkl'
LASSO_PICKLE = '../trained_networks/lasso_data.pkl'

def run_models(x_train, y_train, x_test, y_test):
    """
    Driving all machine learning models as parallel processes.
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """
    build_nn(x_train, y_train, x_test, y_test)
    build_svm(x_train, y_train, x_test, y_test)
    build_tree(x_train, y_train, x_test, y_test)
    build_ridge(x_train, y_train, x_test, y_test)
    build_bayesian_rr(x_train, y_train, x_test, y_test)
    build_lasso(x_train, y_train, x_test, y_test)
    return

def build_nn(x_train, y_train, x_test, y_test):
    """
    Constructing a regression neural network model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """

    # Create regression model
    net = NeuralNet(layers=[('input', InputLayer),
                            ('hidden0', DenseLayer),
                            ('hidden1', DenseLayer),
                            ('output', DenseLayer)],
                    input_shape=(None, x_train.shape[1]),
                    hidden0_num_units=NODES,
                    hidden0_nonlinearity=nonlinearities.softmax,
                    hidden1_num_units=NODES,
                    hidden1_nonlinearity=nonlinearities.softmax,
                    output_num_units=y_train.shape[0],
                    output_nonlinearity=nonlinearities.softmax,
                    update_learning_rate=0.1,
                    regression=True,
                    verbose=1)

    param_grid = {'hidden0_num_units': [1, 4, 17, 25],
                  'hidden1_num_units': [1, 4, 17, 25],
                  'update_learning_rate': [0.01, 0.1, 0.5]}

    # Finding the optimal set of params for each variable in the training of the neural network
    clf = sklearn.model_selection.GridSearchCV(net, param_grid, verbose=0, n_jobs=3, cv=5)
    clf.fit(x_train, y_train)

    y_pred = clf.predict(x_test)
    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)

    with open(NN_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(net, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_svm(x_train, y_train, x_test, y_test):
    """
    Constructing a support vector regression model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """

    clf = SVR(kernel='linear', verbose=True)
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)

    with open(SVM_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_tree(x_train, y_train, x_test, y_test):
    """
    Constructing a decision trees regression model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """
    clf = sklearn.tree.DecisionTreeRegressor()
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)

    with open(DT_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_ridge(x_train, y_train, x_test, y_test):
    """
    Constructing a ridge regression model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """
    clf = RidgeCV(alphas=[0.01, 0.1, 1.0, 10.0], cv=5)
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)
    # Optimal ridge regression alpha value from CV
    ridge_alpha = clf.alpha_

    with open(RR_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(ridge_alpha, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_bayesian_rr(x_train, y_train, x_test, y_test):
    """
    Constructing a Bayesian ridge regression model from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """
    clf = BayesianRidge()
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)
    # Optimal ridge regression alpha value from CV
    ridge_alpha = clf.alpha_

    with open(BRR_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(ridge_alpha, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return

def build_lasso(x_train, y_train, x_test, y_test):
    """
    Constructing a Lasso linear model with cross validation from input dataframe
    :param x_train: features dataframe for model training
    :param y_train: target dataframe for model training
    :param x_test: features dataframe for model testing
    :param y_test: target dataframe for model testing
    :return: None
    """

    clf = LassoCV(tol=0.01, max_iter=10000, cv=5)
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)

    # Mean absolute error regression loss
    mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
    # Mean squared error regression loss
    mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
    # Median absolute error regression loss
    median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
    # R^2 (coefficient of determination) regression score function
    r2 = sklearn.metrics.r2_score(y_test, y_pred)
    # Explained variance regression score function
    exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)
    # Optimal ridge regression alpha value from CV
    lasso_alpha = clf.alpha_

    with open(LASSO_PICKLE, 'wb') as results:
        pickle.dump(clf, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(lasso_alpha, results, pickle.HIGHEST_PROTOCOL)
        pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)

    return
<|endoftext|>


<commit_before>import tensorflow as tf

import sys

sys.path.append('../')
import tfutil as t

tf.set_random_seed(777)  # reproducibility

class InfoGAN:

    def __init__(self, s, batch_size=64, height=32, width=32, channel=3,
                 sample_num=10 * 10, sample_size=10,
                 df_dim=64, gf_dim=64, fc_unit=128, n_categories=10, n_continous_factor=1,
                 z_dim=128, g_lr=1e-3, d_lr=2e-4):

        """
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 64
        :param height: input image height, default 32
        :param width: input image width, default 32
        :param channel: input image channel, default 3 (RGB)

        # Output Settings
        :param sample_num: the number of output images, default 100
        :param sample_size: sample image size, default 10

        # Hyper-parameters
        :param df_dim: discriminator filter, default 64
        :param gf_dim: generator filter, default 64
        :param fc_unit: fully connected unit, default 128

        # Training Option
        :param n_categories: the number of categories, default 10
        :param n_continous_factor: the number of cont factors, default 10
        :param z_dim: z dimension (kinda noise), default 128
        :param g_lr: generator learning rate, default 1e-3
        :param d_lr: discriminator learning rate, default 2e-4
        """

        self.s = s
        self.batch_size = batch_size

        self.height = height
        self.width = width
        self.channel = channel
        self.image_shape = [self.batch_size, self.height, self.width, self.channel]

        self.sample_num = sample_num
        self.sample_size = sample_size

        self.df_dim = df_dim
        self.gf_dim = gf_dim
        self.fc_unit = fc_unit

        """
        - MNIST
        n_cat : 10, n_cont : 2, z : 62 => embeddings : 10 + 2 + 62 = 74
        - SVHN
        n_cat : 10, n_cont : 4, z : 124 => embeddings : 40 + 124 = 168
        - Celeb-A
        n_cat : 10, n_cont : 10, z : 128 => embeddings : 100 + 128 = 228
        """
        self.n_cat = n_categories         # category dist, label
        self.n_cont = n_continous_factor  # gaussian dist, rotate, etc
        self.z_dim = z_dim
        self.lambda_ = 1.  # sufficient for discrete latent codes # less than 1

        self.beta1 = 0.5
        self.beta2 = 0.999
        self.d_lr = d_lr
        self.g_lr = g_lr

        # pre-defined
        self.d_real = 0.
        self.d_fake = 0.

        self.g_loss = 0.
        self.d_adv_loss = 0.
        self.d_loss = 0.

        self.g = None
        self.g_test = None

        self.d_op = None
        self.g_op = None
        self.q_op = None

        self.merged = None
        self.writer = None
        self.saver = None

        # Placeholders
        self.x = tf.placeholder(tf.float32,
                                shape=[None, self.height, self.width, self.channel],
                                name="x-image")                                                     # (-1, 32, 32, 3)
        self.c = tf.placeholder(tf.float32, shape=[None, self.n_cont + self.n_cat], name='c-cond')  # (-1, 11)
        self.z = tf.placeholder(tf.float32, shape=[None, self.z_dim], name='z-noise')               # (-1, 128)

        self.build_infogan()  # build InfoGAN model

    def discriminator(self, x, reuse=None):
        """
        :param x: images
        :param reuse: re-usable
        :return: logits
        """
        with tf.variable_scope("discriminator", reuse=reuse):
            x = t.conv2d(x, self.df_dim * 1, 4, 2, name='disc-conv2d-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = t.conv2d(x, self.df_dim * 2, 4, 2, name='disc-conv2d-2')
            x = t.batch_norm(x, name='disc-bn-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = t.conv2d(x, self.df_dim * 4, 4, 2, name='disc-conv2d-3')
            x = t.batch_norm(x, name='disc-bn-2')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = tf.layers.flatten(x)

            x = t.dense(x, self.fc_unit, name='rec-fc-1')
            x = t.batch_norm(x, name='rec-bn-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = t.dense(x, 1 + self.n_cont + self.n_cat, name='rec-fc-2')
            prob, cont, cat = x[:, 0], x[:, 1:1 + self.n_cont], x[:, 1 + self.n_cont:]  # logits

            prob = tf.nn.sigmoid(prob)  # probability
            cont = cont  # continuous
            cat = tf.nn.softmax(cat)  # categories

            return prob, cont, cat

    def generator(self, z, c, reuse=None, is_train=True):
        """
        :param z: 139 z-noise
        :param c: 10 categories * 10 dimensions
        :param reuse: re-usable
        :param is_train: trainable
        :return: prob
        """
        with tf.variable_scope("generator", reuse=reuse):
            x = tf.concat([z, c], axis=1)  # (-1, 128 + 1 + 10)

            x = t.dense(x, 2 * 2 * 448, name='gen-fc-1')
            x = t.batch_norm(x, is_train=is_train, name='gen-bn-1')
            x = tf.nn.relu(x)

            x = tf.reshape(x, (-1, 2, 2, 448))

            x = t.deconv2d(x, self.gf_dim * 4, 4, 2, name='gen-deconv2d-1')
            x = t.batch_norm(x, is_train=is_train, name='gen-bn-2')
            x = tf.nn.relu(x)

            x = t.deconv2d(x, self.gf_dim * 2, 4, 2, name='gen-deconv2d-2')
            x = tf.nn.relu(x)

            x = t.deconv2d(x, self.gf_dim * 1, 4, 2, name='gen-deconv2d-3')
            x = tf.nn.relu(x)

            x = t.deconv2d(x, 3, 4, 2, name='gen-deconv2d-4')
            x = tf.nn.tanh(x)
            return x

    def build_infogan(self):
        # Generator
        self.g = self.generator(self.z, self.c)
        self.g_test = self.generator(self.z, self.c, is_train=False)

        # Discriminator
        d_real, d_real_cont, d_real_cat = self.discriminator(self.x)
        d_fake, d_fake_cont, d_fake_cat = self.discriminator(self.g, reuse=True)

        # Losses
        d_adv_real_loss = -tf.reduce_mean(t.safe_log(d_real))
        d_adv_fake_loss = -tf.reduce_mean(t.safe_log(1. - d_fake))
        self.d_adv_loss = d_adv_real_loss + d_adv_fake_loss

        d_cont_loss = tf.reduce_mean(tf.square(d_fake_cont / .5))
        cont = self.c[:, self.n_cont:]
        d_cat_loss = -(tf.reduce_mean(tf.reduce_sum(cont * d_fake_cont)) + tf.reduce_mean(cont * cont))

        d_info_loss = self.lambda_ * (d_cont_loss + d_cat_loss)

        self.d_loss = self.d_adv_loss + d_info_loss

        self.g_loss = -tf.reduce_mean(t.safe_log(d_fake)) + d_info_loss

        # Summary
        tf.summary.scalar("loss/d_adv_real_loss", d_adv_real_loss)
        tf.summary.scalar("loss/d_adv_fake_loss", d_adv_fake_loss)
        tf.summary.scalar("loss/d_adv_loss", self.d_adv_loss)
        tf.summary.scalar("loss/d_cont_loss", d_cont_loss)
        tf.summary.scalar("loss/d_cat_loss", d_cat_loss)
        tf.summary.scalar("loss/d_info_loss", d_info_loss)
        tf.summary.scalar("loss/d_loss", self.d_loss)
        tf.summary.scalar("loss/g_loss", self.g_loss)

        # Optimizer
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('d')]
        g_params = [v for v in t_vars if v.name.startswith('g')]

        self.d_op = tf.train.AdamOptimizer(learning_rate=self.d_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.d_loss, var_list=d_params)
        self.g_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.g_loss, var_list=g_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<commit_msg>fix: generator model reuse problem<commit_after>import tensorflow as tf

import sys

sys.path.append('../')
import tfutil as t

tf.set_random_seed(777)  # reproducibility

class InfoGAN:

    def __init__(self, s, batch_size=64, height=32, width=32, channel=3,
                 sample_num=10 * 10, sample_size=10,
                 df_dim=64, gf_dim=64, fc_unit=128, n_categories=10, n_continous_factor=1,
                 z_dim=128, g_lr=1e-3, d_lr=2e-4):

        """
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 64
        :param height: input image height, default 32
        :param width: input image width, default 32
        :param channel: input image channel, default 3 (RGB)

        # Output Settings
        :param sample_num: the number of output images, default 100
        :param sample_size: sample image size, default 10

        # Hyper-parameters
        :param df_dim: discriminator filter, default 64
        :param gf_dim: generator filter, default 64
        :param fc_unit: fully connected unit, default 128

        # Training Option
        :param n_categories: the number of categories, default 10
        :param n_continous_factor: the number of cont factors, default 10
        :param z_dim: z dimension (kinda noise), default 128
        :param g_lr: generator learning rate, default 1e-3
        :param d_lr: discriminator learning rate, default 2e-4
        """

        self.s = s
        self.batch_size = batch_size

        self.height = height
        self.width = width
        self.channel = channel
        self.image_shape = [self.batch_size, self.height, self.width, self.channel]

        self.sample_num = sample_num
        self.sample_size = sample_size

        self.df_dim = df_dim
        self.gf_dim = gf_dim
        self.fc_unit = fc_unit

        """
        - MNIST
        n_cat : 10, n_cont : 2, z : 62 => embeddings : 10 + 2 + 62 = 74
        - SVHN
        n_cat : 10, n_cont : 4, z : 124 => embeddings : 40 + 124 = 168
        - Celeb-A
        n_cat : 10, n_cont : 10, z : 128 => embeddings : 100 + 128 = 228
        """
        self.n_cat = n_categories         # category dist, label
        self.n_cont = n_continous_factor  # gaussian dist, rotate, etc
        self.z_dim = z_dim
        self.lambda_ = 1.  # sufficient for discrete latent codes # less than 1

        self.beta1 = 0.5
        self.beta2 = 0.999
        self.d_lr = d_lr
        self.g_lr = g_lr

        # pre-defined
        self.d_real = 0.
        self.d_fake = 0.

        self.g_loss = 0.
        self.d_adv_loss = 0.
        self.d_loss = 0.

        self.g = None
        self.g_test = None

        self.d_op = None
        self.g_op = None
        self.q_op = None

        self.merged = None
        self.writer = None
        self.saver = None

        # Placeholders
        self.x = tf.placeholder(tf.float32,
                                shape=[None, self.height, self.width, self.channel],
                                name="x-image")                                                     # (-1, 32, 32, 3)
        self.c = tf.placeholder(tf.float32, shape=[None, self.n_cont + self.n_cat], name='c-cond')  # (-1, 11)
        self.z = tf.placeholder(tf.float32, shape=[None, self.z_dim], name='z-noise')               # (-1, 128)

        self.build_infogan()  # build InfoGAN model

    def discriminator(self, x, reuse=None):
        """
        :param x: images
        :param reuse: re-usable
        :return: logits
        """
        with tf.variable_scope("discriminator", reuse=reuse):
            x = t.conv2d(x, self.df_dim * 1, 4, 2, name='disc-conv2d-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = t.conv2d(x, self.df_dim * 2, 4, 2, name='disc-conv2d-2')
            x = t.batch_norm(x, name='disc-bn-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = t.conv2d(x, self.df_dim * 4, 4, 2, name='disc-conv2d-3')
            x = t.batch_norm(x, name='disc-bn-2')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = tf.layers.flatten(x)

            x = t.dense(x, self.fc_unit, name='rec-fc-1')
            x = t.batch_norm(x, name='rec-bn-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = t.dense(x, 1 + self.n_cont + self.n_cat, name='rec-fc-2')
            prob, cont, cat = x[:, 0], x[:, 1:1 + self.n_cont], x[:, 1 + self.n_cont:]  # logits

            prob = tf.nn.sigmoid(prob)  # probability
            cont = cont  # continuous
            cat = tf.nn.softmax(cat)  # categories

            return prob, cont, cat

    def generator(self, z, c, reuse=None, is_train=True):
        """
        :param z: 139 z-noise
        :param c: 10 categories * 10 dimensions
        :param reuse: re-usable
        :param is_train: trainable
        :return: prob
        """
        with tf.variable_scope("generator", reuse=reuse):
            x = tf.concat([z, c], axis=1)  # (-1, 128 + 1 + 10)

            x = t.dense(x, 2 * 2 * 448, name='gen-fc-1')
            x = t.batch_norm(x, is_train=is_train, name='gen-bn-1')
            x = tf.nn.relu(x)

            x = tf.reshape(x, (-1, 2, 2, 448))

            x = t.deconv2d(x, self.gf_dim * 4, 4, 2, name='gen-deconv2d-1')
            x = t.batch_norm(x, is_train=is_train, name='gen-bn-2')
            x = tf.nn.relu(x)

            x = t.deconv2d(x, self.gf_dim * 2, 4, 2, name='gen-deconv2d-2')
            x = tf.nn.relu(x)

            x = t.deconv2d(x, self.gf_dim * 1, 4, 2, name='gen-deconv2d-3')
            x = tf.nn.relu(x)

            x = t.deconv2d(x, 3, 4, 2, name='gen-deconv2d-4')
            x = tf.nn.tanh(x)
            return x

    def build_infogan(self):
        # Generator
        self.g = self.generator(self.z, self.c)
        self.g_test = self.generator(self.z, self.c, reuse=True, is_train=False)

        # Discriminator
        d_real, d_real_cont, d_real_cat = self.discriminator(self.x)
        d_fake, d_fake_cont, d_fake_cat = self.discriminator(self.g, reuse=True)

        # Losses
        d_adv_real_loss = -tf.reduce_mean(t.safe_log(d_real))
        d_adv_fake_loss = -tf.reduce_mean(t.safe_log(1. - d_fake))
        self.d_adv_loss = d_adv_real_loss + d_adv_fake_loss

        d_cont_loss = tf.reduce_mean(tf.square(d_fake_cont / .5))
        cont = self.c[:, self.n_cont:]
        d_cat_loss = -(tf.reduce_mean(tf.reduce_sum(cont * d_fake_cont)) + tf.reduce_mean(cont * cont))

        d_info_loss = self.lambda_ * (d_cont_loss + d_cat_loss)

        self.d_loss = self.d_adv_loss + d_info_loss

        self.g_loss = -tf.reduce_mean(t.safe_log(d_fake)) + d_info_loss

        # Summary
        tf.summary.scalar("loss/d_adv_real_loss", d_adv_real_loss)
        tf.summary.scalar("loss/d_adv_fake_loss", d_adv_fake_loss)
        tf.summary.scalar("loss/d_adv_loss", self.d_adv_loss)
        tf.summary.scalar("loss/d_cont_loss", d_cont_loss)
        tf.summary.scalar("loss/d_cat_loss", d_cat_loss)
        tf.summary.scalar("loss/d_info_loss", d_info_loss)
        tf.summary.scalar("loss/d_loss", self.d_loss)
        tf.summary.scalar("loss/g_loss", self.g_loss)

        # Optimizer
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('d')]
        g_params = [v for v in t_vars if v.name.startswith('g')]

        self.d_op = tf.train.AdamOptimizer(learning_rate=self.d_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.d_loss, var_list=d_params)
        self.g_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.g_loss, var_list=g_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<|endoftext|>


<commit_before><commit_msg>parser for string from auv6, splited by ';'<commit_after>#!/usr/bin/env python

# ROS Imports
import rospy

def parse_from_java(str):
    parsed_tab = str.split(';')
    if len(parsed_tab) != 4:
        rospy.logerror(
            "The string from AUV6 does not contain enought arguments")
        return None
    try:
        parsed_tab[3] = int(parsed_tab[3])
    except ValueError:
        rospy.logerror(
            "The fourth argument has to be an integer")
        return None
    return parsed_tab
<|endoftext|>


<commit_before># -*- Python -*-

# Configuration file for the 'lit' test runner.

import os
import sys
import re
import platform
import subprocess

import lit.util
import lit.formats
from lit.llvm import llvm_config
from lit.llvm.subst import FindTool
from lit.llvm.subst import ToolSubst

# name: The name of this test suite.
config.name = 'LLVM'

# testFormat: The test format to use to interpret tests.
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)

# suffixes: A list of file extensions to treat as test files. This is overriden
# by individual lit.local.cfg files in the test subdirectories.
config.suffixes = ['.ll', '.c', '.cxx', '.test', '.txt', '.s', '.mir']

# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
# subdirectories contain auxiliary inputs for various tests in their parent
# directories.
config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']

# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)

# test_exec_root: The root path where tests should be run.
config.test_exec_root = os.path.join(config.llvm_obj_root, 'test')

# Tweak the PATH to include the tools dir.
llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)

# Propagate some variables from the host environment.
llvm_config.with_system_environment(
    ['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP', 'ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH'])

# Set up OCAMLPATH to include newly built OCaml libraries.
top_ocaml_lib = os.path.join(config.llvm_lib_dir, 'ocaml')
llvm_ocaml_lib = os.path.join(top_ocaml_lib, 'llvm')

llvm_config.with_system_environment('OCAMLPATH')
llvm_config.with_environment('OCAMLPATH', top_ocaml_lib, append_path=True)
llvm_config.with_environment('OCAMLPATH', llvm_ocaml_lib, append_path=True)

llvm_config.with_system_environment('CAML_LD_LIBRARY_PATH')
llvm_config.with_environment(
    'CAML_LD_LIBRARY_PATH', llvm_ocaml_lib, append_path=True)

# Set up OCAMLRUNPARAM to enable backtraces in OCaml tests.
llvm_config.with_environment('OCAMLRUNPARAM', 'b')

# Provide the path to asan runtime lib 'libclang_rt.asan_osx_dynamic.dylib' if
# available. This is darwin specific since it's currently only needed on darwin.

def get_asan_rtlib():
    if not 'Address' in config.llvm_use_sanitizer or \
       not 'Darwin' in config.host_os or \
       not 'x86' in config.host_triple:
        return ''
    try:
        import glob
    except:
        print('glob module not found, skipping get_asan_rtlib() lookup')
        return ''
    # The libclang_rt.asan_osx_dynamic.dylib path is obtained using the relative
    # path from the host cc.
    host_lib_dir = os.path.join(os.path.dirname(config.host_cc), '../lib')
    asan_dylib_dir_pattern = host_lib_dir + \
        '/clang/*/lib/darwin/libclang_rt.asan_osx_dynamic.dylib'
    found_dylibs = glob.glob(asan_dylib_dir_pattern)
    if len(found_dylibs) != 1:
        return ''
    return found_dylibs[0]

llvm_config.use_default_substitutions()

# Add site-specific substitutions.
config.substitutions.append(('%llvmshlibdir', config.llvm_shlib_dir))
config.substitutions.append(('%shlibext', config.llvm_shlib_ext))
config.substitutions.append(('%exeext', config.llvm_exe_ext))

lli_args = []
# The target triple used by default by lli is the process target triple (some
# triple appropriate for generating code for the current process) but because
# we don't support COFF in MCJIT well enough for the tests, force ELF format on
# Windows.  FIXME: the process target triple should be used here, but this is
# difficult to obtain on Windows.
if re.search(r'cygwin|windows-gnu|windows-msvc', config.host_triple):
    lli_args = ['-mtriple=' + config.host_triple + '-elf']

llc_args = []

# Similarly, have a macro to use llc with DWARF even when the host is Windows
if re.search(r'windows-msvc', config.target_triple):
    llc_args = [' -mtriple=' +
                config.target_triple.replace('-msvc', '-gnu')]

# Provide the path to asan runtime lib if available. On darwin, this lib needs
# to be loaded via DYLD_INSERT_LIBRARIES before libLTO.dylib in case the files
# to be linked contain instrumented sanitizer code.
ld64_cmd = config.ld64_executable
asan_rtlib = get_asan_rtlib()
if asan_rtlib:
    ld64_cmd = 'DYLD_INSERT_LIBRARIES={} {}'.format(asan_rtlib, ld64_cmd)

ocamlc_command = '%s ocamlc -cclib -L%s %s' % (
    config.ocamlfind_executable, config.llvm_lib_dir, config.ocaml_flags)
ocamlopt_command = 'true'
if config.have_ocamlopt:
    ocamlopt_command = '%s ocamlopt -cclib -L%s -cclib -Wl,-rpath,%s %s' % (
        config.ocamlfind_executable, config.llvm_lib_dir, config.llvm_lib_dir, config.ocaml_flags)

opt_viewer_cmd = '%s %s/tools/opt-viewer/opt-viewer.py' % (sys.executable, config.llvm_src_root)

tools = [
    ToolSubst('%lli', FindTool('lli'), post='.', extra_args=lli_args),
    ToolSubst('%llc_dwarf', FindTool('llc'), extra_args=llc_args),
    ToolSubst('%go', config.go_executable, unresolved='ignore'),
    ToolSubst('%gold', config.gold_executable, unresolved='ignore'),
    ToolSubst('%ld64', ld64_cmd, unresolved='ignore'),
    ToolSubst('%ocamlc', ocamlc_command, unresolved='ignore'),
    ToolSubst('%ocamlopt', ocamlopt_command, unresolved='ignore'),
    ToolSubst('%opt-viewer', opt_viewer_cmd),
    ToolSubst('%llvm-objcopy', FindTool('llvm-objcopy')),
    ToolSubst('%llvm-strip', FindTool('llvm-strip')),
]

# FIXME: Why do we have both `lli` and `%lli` that do slightly different things?
tools.extend([
    'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as',
    'llvm-bcanalyzer', 'llvm-config', 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres',
    'llvm-diff', 'llvm-dis', 'llvm-dwarfdump', 'llvm-exegesis', 'llvm-extract',
    'llvm-isel-fuzzer', 'llvm-opt-fuzzer', 'llvm-lib', 'llvm-link', 'llvm-lto',
    'llvm-lto2', 'llvm-mc', 'llvm-mca', 'llvm-modextract', 'llvm-nm',
    'llvm-objcopy', 'llvm-objdump', 'llvm-pdbutil', 'llvm-profdata',
    'llvm-ranlib', 'llvm-rc', 'llvm-readelf', 'llvm-readobj', 'llvm-rtdyld', 'llvm-size',
    'llvm-split', 'llvm-strings', 'llvm-strip', 'llvm-tblgen', 'llvm-undname',
    'llvm-c-test', 'llvm-cxxfilt', 'llvm-xray', 'yaml2obj', 'obj2yaml',
    'yaml-bench', 'verify-uselistorder', 'bugpoint', 'llc', 'llvm-symbolizer',
    'opt', 'sancov', 'sanstats'])

# The following tools are optional
tools.extend([
    ToolSubst('llvm-go', unresolved='ignore'),
    ToolSubst('llvm-mt', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch3', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch4', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch5', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch6', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch7', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch8', unresolved='ignore')])

llvm_config.add_tool_substitutions(tools, config.llvm_tools_dir)

# Targets

config.targets = frozenset(config.targets_to_build.split())

for arch in config.targets_to_build.split():
    config.available_features.add(arch.lower() + '-registered-target')

# Features
known_arches = ["x86_64", "mips64", "ppc64", "aarch64"]
if (config.host_ldflags.find("-m32") < 0
    and any(config.llvm_host_triple.startswith(x) for x in known_arches)):
  config.available_features.add("llvm-64-bits")

# Others/can-execute.txt
if sys.platform not in ['win32']:
    config.available_features.add('can-execute')
    config.available_features.add('not_COFF')

# Loadable module
# FIXME: This should be supplied by Makefile or autoconf.
if sys.platform in ['win32', 'cygwin']:
    loadable_module = (config.enable_shared == 1)
else:
    loadable_module = True

if loadable_module:
    config.available_features.add('loadable_module')

# Static libraries are not built if BUILD_SHARED_LIBS is ON.
if not config.build_shared_libs and not config.link_llvm_dylib:
    config.available_features.add('static-libs')

def have_cxx_shared_library():
    readobj_exe = lit.util.which('llvm-readobj', config.llvm_tools_dir)
    if not readobj_exe:
        print('llvm-readobj not found')
        return False

    try:
        readobj_cmd = subprocess.Popen(
            [readobj_exe, '-needed-libs', readobj_exe], stdout=subprocess.PIPE)
    except OSError:
        print('could not exec llvm-readobj')
        return False

    readobj_out = readobj_cmd.stdout.read().decode('ascii')
    readobj_cmd.wait()

    regex = re.compile(r'(libc\+\+|libstdc\+\+|msvcp).*\.(so|dylib|dll)')
    needed_libs = False
    for line in readobj_out.splitlines():
        if 'NeededLibraries [' in line:
            needed_libs = True
        if ']' in line:
            needed_libs = False
        if needed_libs and regex.search(line.lower()):
            return True
    return False

if have_cxx_shared_library():
    config.available_features.add('cxx-shared-library')

# Direct object generation
if not 'hexagon' in config.target_triple:
    config.available_features.add('object-emission')

# LLVM can be configured with an empty default triple
# Some tests are "generic" and require a valid default triple
if config.target_triple:
    config.available_features.add('default_triple')

import subprocess

def have_ld_plugin_support():
    if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'LLVMgold' + config.llvm_shlib_ext)):
        return False

    ld_cmd = subprocess.Popen(
        [config.gold_executable, '--help'], stdout=subprocess.PIPE, env={'LANG': 'C'})
    ld_out = ld_cmd.stdout.read().decode()
    ld_cmd.wait()

    if not '-plugin' in ld_out:
        return False

    # check that the used emulations are supported.
    emu_line = [l for l in ld_out.split('\n') if 'supported emulations' in l]
    if len(emu_line) != 1:
        return False
    emu_line = emu_line[0]
    fields = emu_line.split(':')
    if len(fields) != 3:
        return False
    emulations = fields[2].split()
    if 'elf_x86_64' not in emulations:
        return False
    if 'elf32ppc' in emulations:
        config.available_features.add('ld_emu_elf32ppc')

    ld_version = subprocess.Popen(
        [config.gold_executable, '--version'], stdout=subprocess.PIPE, env={'LANG': 'C'})
    if not 'GNU gold' in ld_version.stdout.read().decode():
        return False
    ld_version.wait()

    return True

if have_ld_plugin_support():
    config.available_features.add('ld_plugin')

def have_ld64_plugin_support():
    if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'libLTO' + config.llvm_shlib_ext)):
        return False

    if config.ld64_executable == '':
        return False

    ld_cmd = subprocess.Popen(
        [config.ld64_executable, '-v'], stderr=subprocess.PIPE)
    ld_out = ld_cmd.stderr.read().decode()
    ld_cmd.wait()

    if 'ld64' not in ld_out or 'LTO' not in ld_out:
        return False

    return True

if have_ld64_plugin_support():
    config.available_features.add('ld64_plugin')

# Ask llvm-config about asserts and global-isel.
llvm_config.feature_config(
    [('--assertion-mode', {'ON': 'asserts'}),
     ('--has-global-isel', {'ON': 'global-isel'})])

if 'darwin' == sys.platform:
    try:
        sysctl_cmd = subprocess.Popen(['sysctl', 'hw.optional.fma'],
                                      stdout=subprocess.PIPE)
    except OSError:
        print('Could not exec sysctl')
    result = sysctl_cmd.stdout.read().decode('ascii')
    if -1 != result.find('hw.optional.fma: 1'):
        config.available_features.add('fma3')
    sysctl_cmd.wait()

# .debug_frame is not emitted for targeting Windows x64.
if not re.match(r'^x86_64.*-(windows-gnu|windows-msvc)', config.target_triple):
    config.available_features.add('debug_frame')

if config.have_libxar:
    config.available_features.add('xar')

if config.enable_threads:
    config.available_features.add('thread_support')

if config.llvm_libxml2_enabled:
    config.available_features.add('libxml2')

if config.have_opt_viewer_modules:
    config.available_features.add('have_opt_viewer_modules')
<commit_msg>[JITLink] Add llvm-jitlink to the list of available tools in lit.<commit_after># -*- Python -*-

# Configuration file for the 'lit' test runner.

import os
import sys
import re
import platform
import subprocess

import lit.util
import lit.formats
from lit.llvm import llvm_config
from lit.llvm.subst import FindTool
from lit.llvm.subst import ToolSubst

# name: The name of this test suite.
config.name = 'LLVM'

# testFormat: The test format to use to interpret tests.
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)

# suffixes: A list of file extensions to treat as test files. This is overriden
# by individual lit.local.cfg files in the test subdirectories.
config.suffixes = ['.ll', '.c', '.cxx', '.test', '.txt', '.s', '.mir']

# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
# subdirectories contain auxiliary inputs for various tests in their parent
# directories.
config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']

# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)

# test_exec_root: The root path where tests should be run.
config.test_exec_root = os.path.join(config.llvm_obj_root, 'test')

# Tweak the PATH to include the tools dir.
llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)

# Propagate some variables from the host environment.
llvm_config.with_system_environment(
    ['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP', 'ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH'])

# Set up OCAMLPATH to include newly built OCaml libraries.
top_ocaml_lib = os.path.join(config.llvm_lib_dir, 'ocaml')
llvm_ocaml_lib = os.path.join(top_ocaml_lib, 'llvm')

llvm_config.with_system_environment('OCAMLPATH')
llvm_config.with_environment('OCAMLPATH', top_ocaml_lib, append_path=True)
llvm_config.with_environment('OCAMLPATH', llvm_ocaml_lib, append_path=True)

llvm_config.with_system_environment('CAML_LD_LIBRARY_PATH')
llvm_config.with_environment(
    'CAML_LD_LIBRARY_PATH', llvm_ocaml_lib, append_path=True)

# Set up OCAMLRUNPARAM to enable backtraces in OCaml tests.
llvm_config.with_environment('OCAMLRUNPARAM', 'b')

# Provide the path to asan runtime lib 'libclang_rt.asan_osx_dynamic.dylib' if
# available. This is darwin specific since it's currently only needed on darwin.

def get_asan_rtlib():
    if not 'Address' in config.llvm_use_sanitizer or \
       not 'Darwin' in config.host_os or \
       not 'x86' in config.host_triple:
        return ''
    try:
        import glob
    except:
        print('glob module not found, skipping get_asan_rtlib() lookup')
        return ''
    # The libclang_rt.asan_osx_dynamic.dylib path is obtained using the relative
    # path from the host cc.
    host_lib_dir = os.path.join(os.path.dirname(config.host_cc), '../lib')
    asan_dylib_dir_pattern = host_lib_dir + \
        '/clang/*/lib/darwin/libclang_rt.asan_osx_dynamic.dylib'
    found_dylibs = glob.glob(asan_dylib_dir_pattern)
    if len(found_dylibs) != 1:
        return ''
    return found_dylibs[0]

llvm_config.use_default_substitutions()

# Add site-specific substitutions.
config.substitutions.append(('%llvmshlibdir', config.llvm_shlib_dir))
config.substitutions.append(('%shlibext', config.llvm_shlib_ext))
config.substitutions.append(('%exeext', config.llvm_exe_ext))

lli_args = []
# The target triple used by default by lli is the process target triple (some
# triple appropriate for generating code for the current process) but because
# we don't support COFF in MCJIT well enough for the tests, force ELF format on
# Windows.  FIXME: the process target triple should be used here, but this is
# difficult to obtain on Windows.
if re.search(r'cygwin|windows-gnu|windows-msvc', config.host_triple):
    lli_args = ['-mtriple=' + config.host_triple + '-elf']

llc_args = []

# Similarly, have a macro to use llc with DWARF even when the host is Windows
if re.search(r'windows-msvc', config.target_triple):
    llc_args = [' -mtriple=' +
                config.target_triple.replace('-msvc', '-gnu')]

# Provide the path to asan runtime lib if available. On darwin, this lib needs
# to be loaded via DYLD_INSERT_LIBRARIES before libLTO.dylib in case the files
# to be linked contain instrumented sanitizer code.
ld64_cmd = config.ld64_executable
asan_rtlib = get_asan_rtlib()
if asan_rtlib:
    ld64_cmd = 'DYLD_INSERT_LIBRARIES={} {}'.format(asan_rtlib, ld64_cmd)

ocamlc_command = '%s ocamlc -cclib -L%s %s' % (
    config.ocamlfind_executable, config.llvm_lib_dir, config.ocaml_flags)
ocamlopt_command = 'true'
if config.have_ocamlopt:
    ocamlopt_command = '%s ocamlopt -cclib -L%s -cclib -Wl,-rpath,%s %s' % (
        config.ocamlfind_executable, config.llvm_lib_dir, config.llvm_lib_dir, config.ocaml_flags)

opt_viewer_cmd = '%s %s/tools/opt-viewer/opt-viewer.py' % (sys.executable, config.llvm_src_root)

tools = [
    ToolSubst('%lli', FindTool('lli'), post='.', extra_args=lli_args),
    ToolSubst('%llc_dwarf', FindTool('llc'), extra_args=llc_args),
    ToolSubst('%go', config.go_executable, unresolved='ignore'),
    ToolSubst('%gold', config.gold_executable, unresolved='ignore'),
    ToolSubst('%ld64', ld64_cmd, unresolved='ignore'),
    ToolSubst('%ocamlc', ocamlc_command, unresolved='ignore'),
    ToolSubst('%ocamlopt', ocamlopt_command, unresolved='ignore'),
    ToolSubst('%opt-viewer', opt_viewer_cmd),
    ToolSubst('%llvm-objcopy', FindTool('llvm-objcopy')),
    ToolSubst('%llvm-strip', FindTool('llvm-strip')),
]

# FIXME: Why do we have both `lli` and `%lli` that do slightly different things?
tools.extend([
    'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as',
    'llvm-bcanalyzer', 'llvm-config', 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres',
    'llvm-diff', 'llvm-dis', 'llvm-dwarfdump', 'llvm-exegesis', 'llvm-extract',
    'llvm-isel-fuzzer', 'llvm-jitlink', 'llvm-opt-fuzzer', 'llvm-lib',
    'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mca',
    'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump',
    'llvm-pdbutil', 'llvm-profdata', 'llvm-ranlib', 'llvm-rc', 'llvm-readelf',
    'llvm-readobj', 'llvm-rtdyld', 'llvm-size', 'llvm-split', 'llvm-strings',
    'llvm-strip', 'llvm-tblgen', 'llvm-undname', 'llvm-c-test', 'llvm-cxxfilt',
    'llvm-xray', 'yaml2obj', 'obj2yaml', 'yaml-bench', 'verify-uselistorder',
    'bugpoint', 'llc', 'llvm-symbolizer', 'opt', 'sancov', 'sanstats'])

# The following tools are optional
tools.extend([
    ToolSubst('llvm-go', unresolved='ignore'),
    ToolSubst('llvm-mt', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch3', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch4', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch5', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch6', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch7', unresolved='ignore'),
    ToolSubst('Kaleidoscope-Ch8', unresolved='ignore')])

llvm_config.add_tool_substitutions(tools, config.llvm_tools_dir)

# Targets

config.targets = frozenset(config.targets_to_build.split())

for arch in config.targets_to_build.split():
    config.available_features.add(arch.lower() + '-registered-target')

# Features
known_arches = ["x86_64", "mips64", "ppc64", "aarch64"]
if (config.host_ldflags.find("-m32") < 0
    and any(config.llvm_host_triple.startswith(x) for x in known_arches)):
  config.available_features.add("llvm-64-bits")

# Others/can-execute.txt
if sys.platform not in ['win32']:
    config.available_features.add('can-execute')
    config.available_features.add('not_COFF')

# Loadable module
# FIXME: This should be supplied by Makefile or autoconf.
if sys.platform in ['win32', 'cygwin']:
    loadable_module = (config.enable_shared == 1)
else:
    loadable_module = True

if loadable_module:
    config.available_features.add('loadable_module')

# Static libraries are not built if BUILD_SHARED_LIBS is ON.
if not config.build_shared_libs and not config.link_llvm_dylib:
    config.available_features.add('static-libs')

def have_cxx_shared_library():
    readobj_exe = lit.util.which('llvm-readobj', config.llvm_tools_dir)
    if not readobj_exe:
        print('llvm-readobj not found')
        return False

    try:
        readobj_cmd = subprocess.Popen(
            [readobj_exe, '-needed-libs', readobj_exe], stdout=subprocess.PIPE)
    except OSError:
        print('could not exec llvm-readobj')
        return False

    readobj_out = readobj_cmd.stdout.read().decode('ascii')
    readobj_cmd.wait()

    regex = re.compile(r'(libc\+\+|libstdc\+\+|msvcp).*\.(so|dylib|dll)')
    needed_libs = False
    for line in readobj_out.splitlines():
        if 'NeededLibraries [' in line:
            needed_libs = True
        if ']' in line:
            needed_libs = False
        if needed_libs and regex.search(line.lower()):
            return True
    return False

if have_cxx_shared_library():
    config.available_features.add('cxx-shared-library')

# Direct object generation
if not 'hexagon' in config.target_triple:
    config.available_features.add('object-emission')

# LLVM can be configured with an empty default triple
# Some tests are "generic" and require a valid default triple
if config.target_triple:
    config.available_features.add('default_triple')

import subprocess

def have_ld_plugin_support():
    if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'LLVMgold' + config.llvm_shlib_ext)):
        return False

    ld_cmd = subprocess.Popen(
        [config.gold_executable, '--help'], stdout=subprocess.PIPE, env={'LANG': 'C'})
    ld_out = ld_cmd.stdout.read().decode()
    ld_cmd.wait()

    if not '-plugin' in ld_out:
        return False

    # check that the used emulations are supported.
    emu_line = [l for l in ld_out.split('\n') if 'supported emulations' in l]
    if len(emu_line) != 1:
        return False
    emu_line = emu_line[0]
    fields = emu_line.split(':')
    if len(fields) != 3:
        return False
    emulations = fields[2].split()
    if 'elf_x86_64' not in emulations:
        return False
    if 'elf32ppc' in emulations:
        config.available_features.add('ld_emu_elf32ppc')

    ld_version = subprocess.Popen(
        [config.gold_executable, '--version'], stdout=subprocess.PIPE, env={'LANG': 'C'})
    if not 'GNU gold' in ld_version.stdout.read().decode():
        return False
    ld_version.wait()

    return True

if have_ld_plugin_support():
    config.available_features.add('ld_plugin')

def have_ld64_plugin_support():
    if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'libLTO' + config.llvm_shlib_ext)):
        return False

    if config.ld64_executable == '':
        return False

    ld_cmd = subprocess.Popen(
        [config.ld64_executable, '-v'], stderr=subprocess.PIPE)
    ld_out = ld_cmd.stderr.read().decode()
    ld_cmd.wait()

    if 'ld64' not in ld_out or 'LTO' not in ld_out:
        return False

    return True

if have_ld64_plugin_support():
    config.available_features.add('ld64_plugin')

# Ask llvm-config about asserts and global-isel.
llvm_config.feature_config(
    [('--assertion-mode', {'ON': 'asserts'}),
     ('--has-global-isel', {'ON': 'global-isel'})])

if 'darwin' == sys.platform:
    try:
        sysctl_cmd = subprocess.Popen(['sysctl', 'hw.optional.fma'],
                                      stdout=subprocess.PIPE)
    except OSError:
        print('Could not exec sysctl')
    result = sysctl_cmd.stdout.read().decode('ascii')
    if -1 != result.find('hw.optional.fma: 1'):
        config.available_features.add('fma3')
    sysctl_cmd.wait()

# .debug_frame is not emitted for targeting Windows x64.
if not re.match(r'^x86_64.*-(windows-gnu|windows-msvc)', config.target_triple):
    config.available_features.add('debug_frame')

if config.have_libxar:
    config.available_features.add('xar')

if config.enable_threads:
    config.available_features.add('thread_support')

if config.llvm_libxml2_enabled:
    config.available_features.add('libxml2')

if config.have_opt_viewer_modules:
    config.available_features.add('have_opt_viewer_modules')
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

#
# Interface to the AVT Vimba SDK
#

#
# Inspired from :
#
#       - Pymba
#         https://github.com/morefigs/pymba
#

#
# External dependencies
#
import os
import sys
import time
import ctypes as ct
import numpy as np

#
# Global access to the Vimba library
#
vimba = None

#
# Initialize the Vimba library
#
def VmbStartup() :

	# Get Vimba installation directory
	vimba_path = '/' + '/'.join( os.environ.get( 'GENICAM_GENTL64_PATH' ).split( '/' )[ 1 : -3 ] )
	vimba_path += '/VimbaC/DynamicLib/x86_64bit/libVimbaC.so'

	# Load Vimba library
	global vimba
	vimba = ct.cdll.LoadLibrary( vimba_path )

	# Initialize the library
	vimba.VmbStartup()

	# Send discovery packet to GigE cameras
	vimba.VmbFeatureCommandRun( ct.c_void_p(1), 'GeVDiscoveryAllOnce' )

#
# Release the Vimba library
#
def VmbShutdown() :

	# Release the library
	vimba.VmbShutdown()

#
# Vimba frame structure
#
class VmbFrame( ct.Structure ) :

	#
	# VmbFrame structure fields
	#
	_fields_ = [ ( 'buffer', ct.POINTER( ct.c_char ) ),
			( 'bufferSize', ct.c_uint32 ),
			( 'context', ct.c_void_p * 4 ),
			( 'receiveStatus', ct.c_int32 ),
			( 'receiveFlags', ct.c_uint32 ),
			( 'imageSize', ct.c_uint32 ),
			( 'ancillarySize', ct.c_uint32 ),
			( 'pixelFormat', ct.c_uint32 ),
			( 'width', ct.c_uint32 ),
			( 'height', ct.c_uint32 ),
			( 'offsetX', ct.c_uint32 ),
			( 'offsetY', ct.c_uint32 ),
			( 'frameID', ct.c_uint64 ),
			( 'timestamp', ct.c_uint64 ) ]

	#
	# Initialize the image buffer
	#
	def __init__( self, frame_size ) :

		self.buffer = ct.create_string_buffer( frame_size )
		self.bufferSize = ct.c_uint32( frame_size )

	#
	# Convert the frame to a numpy array
	#
	@property
	def image( self ) :

		return np.ndarray( buffer=self.buffer[ 0 : self.bufferSize ], dtype=np.uint8, shape=( self.height, self.width ) )

	#
	# Tell if the frame is valid
	#
	@property
	def is_valid( self ) :

		return not self.receiveStatus

#
# Vimba camera
#
class VmbCamera( object ) :

	#
	# Initialize the camera
	#
	def __init__( self, camera_id ) :

		# Camera handle
		self.handle = ct.c_void_p()

		# Camera ID (serial number, IP address...)
		self.id = camera_id

	#
	# Open the camera
	#
	def Open( self ) :

		# Connect the camera
		vimba.VmbCameraOpen( self.id, 1, ct.byref( self.handle ) )

		# Adjust packet size automatically
		vimba.VmbFeatureCommandRun( self.handle, 'GVSPAdjustPacketSize' )

		# Query image parameters
		tmp_value = ct.c_int()
		vimba.VmbFeatureIntGet( self.handle, 'Width', ct.byref( tmp_value ) )
		self.width = tmp_value.value
		vimba.VmbFeatureIntGet( self.handle, 'Height', ct.byref( tmp_value ) )
		self.height = tmp_value.value
		vimba.VmbFeatureIntGet( self.handle, 'PayloadSize', ct.byref( tmp_value ) )
		self.payloadsize = tmp_value.value

	#
	# Close the camera
	#
	def Close( self ) :

		# Close the camera
		vimba.VmbCameraClose( self.handle )

	#
	# Start the acquisition
	#
	def StartCapture( self, frame_callback, frame_buffer_size = 10 ) :

		# Register the external frame callback function
		self.frame_callback = frame_callback

		# Register the internal frame callback function
		self.vmb_frame_callback = ct.CFUNCTYPE( None, ct.c_void_p, ct.POINTER(VmbFrame) )( self.VmbFrameCallback )

		# Initialize frame buffer
		self.frame_buffer = []
		for i in range( frame_buffer_size ) :
			self.frame_buffer.append( VmbFrame( self.payloadsize ) )

		# Announce the frames
		for i in range( frame_buffer_size ) :
			vimba.VmbFrameAnnounce( self.handle, ct.byref(self.frame_buffer[i]), ct.sizeof(self.frame_buffer[i]) )

		# Start capture engine
		vimba.VmbCaptureStart( self.handle )

		# Queue the frames
		for i in range( frame_buffer_size ) :
			vimba.VmbCaptureFrameQueue( self.handle, ct.byref(self.frame_buffer[i]), self.vmb_frame_callback )

		# Start acquisition
		vimba.VmbFeatureCommandRun( self.handle, 'AcquisitionStart' )

	#
	# Stop the acquisition
	#
	def StopCapture( self ) :

		# Stop acquisition
		vimba.VmbFeatureCommandRun( self.handle, 'AcquisitionStop' )

		# Flush the frame queue
		vimba.VmbCaptureQueueFlush( self.handle )

		# Stop capture engine
		vimba.VmbCaptureEnd( self.handle )

		# Revoke frames
		vimba.VmbFrameRevokeAll( self.handle )

	#
	# Function called by Vimba to receive the frame
	#
	def VmbFrameCallback( self, camera, frame ) :

		# Call external frame callback function
		self.frame_callback( frame.contents )

		# Requeue the frame so it can be filled again
		vimba.VmbCaptureFrameQueue( camera, frame, self.vmb_frame_callback )

#
# Vimba stereo camera
#
class VmbStereoCamera( object ) :

	#
	# Initialize the cameras
	#
	def __init__( self, camera_left_id, camera_right_id ) :

		# Camera connection
		self.camera_left = VmbCamera( camera_left_id )
		self.camera_right = VmbCamera( camera_right_id )

	#
	# Open the cameras
	#
	def Open( self ) :

		# Open the cameras
		self.camera_left.Open()
		self.camera_right.Open()

	#
	# Close the cameras
	#
	def Close( self ) :

		# Close the cameras
		self.camera_left.Close()
		self.camera_right.Close()

	#
	# Start synchronous acquisition
	#
	def StartCapture( self, frame_callback ) :

		# Register the external image callback function
		self.frame_callback = frame_callback

		# Initialize frame status
		self.frame_left_time = 0
		self.frame_right_time = 0

		# Start acquisition
		self.camera_left.StartCapture( self.FrameCallbackLeft )
		self.camera_right.StartCapture( self.FrameCallbackRight )

	#
	# Stop the acquisition
	#
	def StopCapture( self ) :

		# Stop image acquisition
		self.camera_left.StopCapture()
		self.camera_right.StopCapture()

	#
	# Receive a frame from camera left
	#
	def FrameCallbackLeft( self, frame ) :

		# Frame time
		self.frame_left_time = time.time()

		# Save the current frame
		self.frame_left = frame

		# Synchronize the frames
		self.Synchronize()

	#
	# Receive a frame from camera right
	#
	def FrameCallbackRight( self, frame ) :

		# Frame time
		self.frame_right_time = time.time()

		# Save the current frame
		self.frame_right = frame

		# Synchronize the frames
		self.Synchronize()

	#
	# Synchronize the images from both camera
	#
	def Synchronize( self ) :

		# Wait for both images
		if self.frame_left_time and self.frame_right_time :

			print( '{}({}) - {}({}) - {}'.format( self.frame_left.frameID, self.frame_left.receiveStatus, self.frame_right.frameID, self.frame_right.receiveStatus, abs( self.frame_left_time - self.frame_right_time ) ) )

			# Check frame time difference
			if self.frame_left.is_valid and self.frame_right.is_valid : # and abs( self.frame_left_time - self.frame_right_time ) < 0.01 :

				# Send the images to the external program
				self.frame_callback( self.frame_left, self.frame_right )

				# Initialize image status
				self.frame_left_time = 0
				self.frame_right_time = 0
<commit_msg>Remove debug message, and cleanup the code.<commit_after># -*- coding:utf-8 -*-

#
# Interface to the AVT Vimba SDK
#

# Inspired from :
#       - Pymba
#         https://github.com/morefigs/pymba

# External dependencies
import os
import sys
import time
import ctypes as ct
import numpy as np

# Global access to the Vimba library
vimba = None

# Initialize the Vimba library
def VmbStartup() :
	# Get Vimba installation directory
	vimba_path = '/' + '/'.join( os.environ.get( 'GENICAM_GENTL64_PATH' ).split( '/' )[ 1 : -3 ] )
	vimba_path += '/VimbaC/DynamicLib/x86_64bit/libVimbaC.so'
	# Load Vimba library
	global vimba
	vimba = ct.cdll.LoadLibrary( vimba_path )
	# Initialize the library
	vimba.VmbStartup()
	# Send discovery packet to GigE cameras
	vimba.VmbFeatureCommandRun( ct.c_void_p(1), 'GeVDiscoveryAllOnce' )

# Release the Vimba library
def VmbShutdown() :
	# Release the library
	vimba.VmbShutdown()

# Vimba frame structure
class VmbFrame( ct.Structure ) :

	# VmbFrame structure fields
	_fields_ = [ ( 'buffer', ct.POINTER( ct.c_char ) ),
			( 'bufferSize', ct.c_uint32 ),
			( 'context', ct.c_void_p * 4 ),
			( 'receiveStatus', ct.c_int32 ),
			( 'receiveFlags', ct.c_uint32 ),
			( 'imageSize', ct.c_uint32 ),
			( 'ancillarySize', ct.c_uint32 ),
			( 'pixelFormat', ct.c_uint32 ),
			( 'width', ct.c_uint32 ),
			( 'height', ct.c_uint32 ),
			( 'offsetX', ct.c_uint32 ),
			( 'offsetY', ct.c_uint32 ),
			( 'frameID', ct.c_uint64 ),
			( 'timestamp', ct.c_uint64 ) ]

	# Initialize the image buffer
	def __init__( self, frame_size ) :
		self.buffer = ct.create_string_buffer( frame_size )
		self.bufferSize = ct.c_uint32( frame_size )

	# Convert the frame to a numpy array
	@property
	def image( self ) :
		return np.ndarray( buffer=self.buffer[ 0 : self.bufferSize ], dtype=np.uint8, shape=( self.height, self.width ) )

	# Tell if the frame is valid
	@property
	def is_valid( self ) :
		return not self.receiveStatus

# Vimba camera
class VmbCamera( object ) :

	# Initialize the camera
	def __init__( self, camera_id ) :
		# Camera handle
		self.handle = ct.c_void_p()
		# Camera ID (serial number, IP address...)
		self.id = camera_id

	# Open the camera
	def Open( self ) :
		# Connect the camera
		vimba.VmbCameraOpen( self.id, 1, ct.byref( self.handle ) )
		# Adjust packet size automatically
		vimba.VmbFeatureCommandRun( self.handle, 'GVSPAdjustPacketSize' )
		# Query image parameters
		tmp_value = ct.c_int()
		vimba.VmbFeatureIntGet( self.handle, 'Width', ct.byref( tmp_value ) )
		self.width = tmp_value.value
		vimba.VmbFeatureIntGet( self.handle, 'Height', ct.byref( tmp_value ) )
		self.height = tmp_value.value
		vimba.VmbFeatureIntGet( self.handle, 'PayloadSize', ct.byref( tmp_value ) )
		self.payloadsize = tmp_value.value

	# Close the camera
	def Close( self ) :
		# Close the camera
		vimba.VmbCameraClose( self.handle )

	# Start the acquisition
	def StartCapture( self, frame_callback, frame_buffer_size = 10 ) :
		# Register the external frame callback function
		self.frame_callback = frame_callback
		# Register the internal frame callback function
		self.vmb_frame_callback = ct.CFUNCTYPE( None, ct.c_void_p, ct.POINTER(VmbFrame) )( self.VmbFrameCallback )
		# Initialize frame buffer
		self.frame_buffer = []
		for i in range( frame_buffer_size ) :
			self.frame_buffer.append( VmbFrame( self.payloadsize ) )
		# Announce the frames
		for i in range( frame_buffer_size ) :
			vimba.VmbFrameAnnounce( self.handle, ct.byref(self.frame_buffer[i]), ct.sizeof(self.frame_buffer[i]) )
		# Start capture engine
		vimba.VmbCaptureStart( self.handle )
		# Queue the frames
		for i in range( frame_buffer_size ) :
			vimba.VmbCaptureFrameQueue( self.handle, ct.byref(self.frame_buffer[i]), self.vmb_frame_callback )
		# Start acquisition
		vimba.VmbFeatureCommandRun( self.handle, 'AcquisitionStart' )

	# Stop the acquisition
	def StopCapture( self ) :
		# Stop acquisition
		vimba.VmbFeatureCommandRun( self.handle, 'AcquisitionStop' )
		# Flush the frame queue
		vimba.VmbCaptureQueueFlush( self.handle )
		# Stop capture engine
		vimba.VmbCaptureEnd( self.handle )
		# Revoke frames
		vimba.VmbFrameRevokeAll( self.handle )

	# Function called by Vimba to receive the frame
	def VmbFrameCallback( self, camera, frame ) :
		# Call external frame callback function
		self.frame_callback( frame.contents )
		# Requeue the frame so it can be filled again
		vimba.VmbCaptureFrameQueue( camera, frame, self.vmb_frame_callback )

# Vimba stereo camera
class VmbStereoCamera( object ) :

	# Initialize the cameras
	def __init__( self, camera_left_id, camera_right_id ) :
		# Camera connection
		self.camera_left = VmbCamera( camera_left_id )
		self.camera_right = VmbCamera( camera_right_id )

	# Open the cameras
	def Open( self ) :
		# Open the cameras
		self.camera_left.Open()
		self.camera_right.Open()

	# Close the cameras
	def Close( self ) :
		# Close the cameras
		self.camera_left.Close()
		self.camera_right.Close()

	# Start synchronous acquisition
	def StartCapture( self, frame_callback ) :
		# Register the external image callback function
		self.frame_callback = frame_callback
		# Initialize the frames
		self.frame_left = None
		self.frame_right = None
		# Start acquisition
		self.camera_left.StartCapture( self.FrameCallbackLeft )
		self.camera_right.StartCapture( self.FrameCallbackRight )

	# Stop the acquisition
	def StopCapture( self ) :
		# Stop image acquisition
		self.camera_left.StopCapture()
		self.camera_right.StopCapture()

	# Receive a frame from camera left
	def FrameCallbackLeft( self, frame ) :
		# Save the current frame
		self.frame_left = frame
		# Synchronize the frames
		self.Synchronize()

	# Receive a frame from camera right
	def FrameCallbackRight( self, frame ) :
		# Save the current frame
		self.frame_right = frame
		# Synchronize the frames
		self.Synchronize()

	# Synchronize the images from both camera
	def Synchronize( self ) :
		# Wait for both images
		if self.frame_left and self.frame_right :
			# Send the images to the external program
			self.frame_callback( self.frame_left, self.frame_right )
			# Initialize the frames
			self.frame_left = None
			self.frame_right = None
<|endoftext|>


<commit_before>from typing import List

from .exceptions import ParserException
from .operations import TOKENS

class Statement(object):

    def __init__(self, operation, arguments: list):
        self.operation = operation
        self.arguments = arguments

    def execute(self):
        self.operation.execute(self.arguments)

class Parser(object):

    def parse_line(self, line: str) -> Statement:
        segments = line.split(" ")
        if len(segments) < 2:
            raise ParserException(f"Statement must consist of 2 parts, not {len(segments)}")

        operation = segments[0]
        if operation not in TOKENS:
            raise ParserException(f"Operation {operation} undefined")

        arguments = "".join(segments[1:]).split(",")

        op = TOKENS[operation]

        return Statement(op, arguments)

    def parse_file(self, filename: str) -> List[Statement]:
        statements = []
        with open(filename) as f:
            for line in f:
                statements.append(self.parse_line(line))

        return statements
<commit_msg>Added type checking for arguments, and added parse_stream method<commit_after>from typing import List
import ast
import io

from .exceptions import ParserException
from .operations import TOKENS

class Statement(object):

    def __init__(self, operation, arguments: list):
        self.operation = operation
        self.arguments = arguments

    def execute(self):
        self.operation.execute(self.arguments)

class Parser(object):

    def parse_line(self, line: str) -> Statement:
        segments = line.split(" ")
        if len(segments) < 2:
            raise ParserException(f"Statement must consist of 2 parts, not {len(segments)}")

        operation = segments[0]
        if operation not in TOKENS:
            raise ParserException(f"Operation {operation} undefined")

        argument_list = "".join(segments[1:]).split(",")
        arguments = list(map(ast.literal_eval, argument_list))
        for argument in arguments:
            if not isinstance(argument, int):
                raise ParserException(f"Statement arguments must be int, not {type(argument)}")

        op = TOKENS[operation]

        return Statement(op, arguments)

    def parse_file(self, filename: str) -> List[Statement]:
        with open(filename) as f:
            return self.parse_stream(f)

    def parse_stream(self, stream: io.IOBase) -> List[Statement]:
        statements = []
        lines = list(map(lambda x: x.strip(), stream.readlines()))
        for line in lines:
            statements.append(self.parse_line(line))

        return statements
<|endoftext|>