<commit_before>#!/usr/bin/env python

"""
Create inputs and perform data manipulation tasks in project workflow
"""

import os
try:
    import urllib.request as urllib2
except ImportError:
    import urllib2

import boto
import matplotlib
matplotlib.use('Agg')  # Must be before importing matplotlib.pyplot or pylab
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn
import math
import boto.s3
from boto.s3.key import Key
from sklearn.ensemble import RandomForestRegressor

__author__ = "Pearl Philip"
__credits__ = "David Beck"
__license__ = "BSD 3-Clause License"
__maintainer__ = "Pearl Philip"
__email__ = "pphilip@uw.edu"
__status__ = "Development"

def create_notation_dataframe(filename):
    """
    Returning Pandas dataframe of sample ID and molecular notation.
    :param filename: File object containing molecular notation indexed by sample ID
    :return: Dataframe of molecular notation indexed by sample ID.
    """
    df = []
    for line in filename:
        # Splits the line into it's key and molecular string
        words = line.split()
        z = [int(words[0]), words[1]]
        df.append(z)
    df = pd.DataFrame(df)
    df.columns = ['CID', 'SMILES']
    df.sort_values(by='CID', inplace=True)
    return df

def create_activity_dataframe(dataframe):
    """
    Performing useful transformations on the acquired data for use in subsequent algorithm.
    :param dataframe: Dataframe downloaded from NCBI database.
    :return: df: Cleaned and sorted dataframe.
    """

    # Eliminates first five text rows of csv
    for j in range(5):
        df = dataframe.drop(j, axis=0)
    df = df.drop(['PUBCHEM_ACTIVITY_URL', 'PUBCHEM_RESULT_TAG',
                  'PUBCHEM_ACTIVITY_SCORE', 'PUBCHEM_SID',
                  'PUBCHEM_ASSAYDATA_COMMENT', 'Potency',
                  'Efficacy', 'Analysis Comment',
                  'Curve_Description', 'Fit_LogAC50',
                  'Fit_HillSlope', 'Fit_R2', 'Fit_ZeroActivity',
                  'Fit_CurveClass', 'Excluded_Points', 'Compound QC',
                  'Max_Response', 'Phenotype', 'Activity at 0.457 uM',
                  'Activity at 2.290 uM', 'Activity at 11.40 uM',
                  'Activity at 57.10 uM', 'PUBCHEM_ACTIVITY_OUTCOME',
                  'Fit_InfiniteActivity'], axis=1)
    df.rename(columns={'PUBCHEM_CID': 'CID'}, inplace=True)

    # Eliminates duplicate compound rows
    df['dupes'] = df.duplicated('CID')
    df = df[df['dupes'] == 0].drop(['dupes'], axis=1)
    df = df.sort_values(by='CID')
    return df

def upload_to_s3(aws_access_key_id, aws_secret_access_key, file_to_s3, bucket, key, callback=None, md5=None,
                 reduced_redundancy=False, content_type=None):
    """
    Uploads the given file to the AWS S3 bucket and key specified.
    :param aws_access_key_id: First part of AWS access key.
    :param aws_secret_access_key: Second part of AWS access key.
    :param file_to_s3: File object to be uploaded.
    :param bucket: S3 bucket name as string.
    :param key: Name attribute of the file object to be uploaded.
    :param callback: Function accepts two integer parameters, the first representing the number of bytes that have been
    successfully transmitted to S3 and the second representing the size of the to be transmitted object. Returns
    boolean indicating success/failure of upload.
    :param md5: MD5 checksum value to verify the integrity of the object.
    :param reduced_redundancy: S3 option that enables customers to reduce their costs
    by storing noncritical, reproducible data at lower levels of redundancy than S3's standard storage.
    :param content_type: Set the type of content in file object.
    :return: Boolean indicating success of upload.
    """
    try:
        size = os.fstat(file_to_s3.fileno()).st_size
    except:
        # Not all file objects implement fileno(), so we fall back on this
        file_to_s3.seek(0, os.SEEK_END)
        size = file_to_s3.tell()

    conn = boto.connect_s3(aws_access_key_id, aws_secret_access_key)
    bucket = conn.get_bucket(bucket, validate=True)
    k = Key(bucket)
    k.key = key
    if content_type:
        k.set_metadata('Content-Type', content_type)
    sent = k.set_contents_from_file(file_to_s3, cb=callback, md5=md5,
                                    reduced_redundancy=reduced_redundancy, rewind=True)

    # Rewind for later use
    file_to_s3.seek(0)

    if sent == size:
        return True
    return False

def join_dataframes():
    """
    Joining the dataframes of existing descriptor files from their urls into a single dataframe.
    :return: Dataframe after join over key column.
    """

    url_list = ['https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_constitution.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_con.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_kappa.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_estate.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_basak.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_property.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_charge.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_moe.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_burden.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_geary.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_moran.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_topology.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_geometric.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_cpsa.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_rdf.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_morse.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_whim.csv'
                ]

    url_exist_list = []
    for url in url_list:
        try:
            r = urllib2.urlopen(url)
        except urllib2.URLError as e:
            r = e
        if r.code < 400:
            url_exist_list.append(url)

    i = 0
    df = [0] * len(url_exist_list)
    for url in url_exist_list:
        df[i] = pd.read_csv(url)

        df[i].drop(df[i].columns[0], axis=1, inplace=True)
        df[i].reset_index(drop=True, inplace=True)
        i += 1

    joined_df = df[0]
    for i in df[1:]:
        joined_df = joined_df.join(i)
    return joined_df

def choose_features(x, y):
    """
    Selecting the features of high importance to reduce feature space.
    :param x: Dataframe of features.
    :param y: Dataframe of target property.
    :return desired x: Dataframe of short-listed features.
    """

    # Random forest feature importance
    x = np.array(x)
    y = np.array(y)
    n_features = x.shape[1]
    max_features = math.ceil(math.sqrt(n_features))
    max_features = max(1, max_features)
    clf = RandomForestRegressor(n_jobs=-1, random_state=1, oob_score=True,
                                max_features=max_features)
    sfm = sklearn.feature_selection.SelectFromModel(clf)
    sfm.fit(x, y)
    desired_x = sfm.transform(x)
    coefficients = sfm.get_support()
    importance = sfm.feature_importances_
    std = np.std([tree.feature_importances_ for tree in sfm.estimators_],
                 axis=0)
    indices = np.argsort(importance)[::-1]

    # Plot the importance of features in the forest
    plt.figure()
    plt.title("Plot of feature importance")
    plt.bar(range(x.shape[1]), importance[indices],
            color="r", yerr=std[indices], align="center")
    plt.xticks(range(x.shape[1]), indices)
    plt.xlim([-1, x.shape[1]])
    plt.savefig('../plots/feature_importance.png')

    return desired_x, coefficients

def change_nan_infinite(dataframe):
    """
    Replacing NaN and infinite values from the dataframe with zeros.
    :param dataframe: Dataframe containing NaN and infinite values.
    :return dataframe: Dataframe with no NaN or infinite values.
    """

    dataframe.replace([np.inf, -np.inf], np.nan, inplace=True)
    dataframe.fillna(0, inplace=True)

    return dataframe

def plot_features(df_x, df_y):
    """
    Plotting each feature x and its corresponding value of target function y.
    :param df_x: Dataframe containing feature space.
    :param df_y: Dataframe containing target/output.
    """
    for column in df_x:
        plt.scatter(df_x[column], df_y)
        plt.title('%s effect on inhibition activity score trend' % df_x[column].name)
        plt.xlabel('%s' % df_x[column].name)
        plt.ylabel('Activity score')
        plt.savefig('../plots/feature_plots/%s.png' % df_x[column].name, bbox_inches='tight')

    return
<commit_msg>Added separate x_train and x_test transforms using selected features, added randomized lasso script<commit_after>#!/usr/bin/env python

"""
Create inputs and perform data manipulation tasks in project workflow
"""

import os
try:
    import urllib.request as urllib2
except ImportError:
    import urllib2

import boto
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pickle
import boto.s3
from boto.s3.key import Key
from sklearn.ensemble import RandomForestRegressor
from sklearn.feature_selection import SelectFromModel
from sklearn.linear_model import RandomizedLasso

__author__ = "Pearl Philip"
__credits__ = "David Beck"
__license__ = "BSD 3-Clause License"
__maintainer__ = "Pearl Philip"
__email__ = "pphilip@uw.edu"
__status__ = "Development"

def create_notation_dataframe(filename):
    """
    Returning Pandas dataframe of sample ID and molecular notation.
    :param filename: File object containing molecular notation indexed by sample ID
    :return: Dataframe of molecular notation indexed by sample ID.
    """
    df = []
    for line in filename:
        # Splits the line into it's key and molecular string
        words = line.split()
        z = [int(words[0]), words[1]]
        df.append(z)
    df = pd.DataFrame(df)
    df.columns = ['CID', 'SMILES']
    df.sort_values(by='CID', inplace=True)
    return df

def create_activity_dataframe(dataframe):
    """
    Performing useful transformations on the acquired data for use in subsequent algorithm.
    :param dataframe: Dataframe downloaded from NCBI database.
    :return: df: Cleaned and sorted dataframe.
    """

    # Eliminates first five text rows of csv
    for j in range(5):
        df = dataframe.drop(j, axis=0)
    df = df.drop(['PUBCHEM_ACTIVITY_URL', 'PUBCHEM_RESULT_TAG',
                  'PUBCHEM_ACTIVITY_SCORE', 'PUBCHEM_SID',
                  'PUBCHEM_ASSAYDATA_COMMENT', 'Potency',
                  'Efficacy', 'Analysis Comment',
                  'Curve_Description', 'Fit_LogAC50',
                  'Fit_HillSlope', 'Fit_R2', 'Fit_ZeroActivity',
                  'Fit_CurveClass', 'Excluded_Points', 'Compound QC',
                  'Max_Response', 'Phenotype', 'Activity at 0.457 uM',
                  'Activity at 2.290 uM', 'Activity at 11.40 uM',
                  'Activity at 57.10 uM', 'PUBCHEM_ACTIVITY_OUTCOME',
                  'Fit_InfiniteActivity'], axis=1)
    df.rename(columns={'PUBCHEM_CID': 'CID'}, inplace=True)

    # Eliminates duplicate compound rows
    df['dupes'] = df.duplicated('CID')
    df = df[df['dupes'] == 0].drop(['dupes'], axis=1)
    df = df.sort_values(by='CID')
    return df

def upload_to_s3(aws_access_key_id, aws_secret_access_key, file_to_s3, bucket, key, callback=None, md5=None,
                 reduced_redundancy=False, content_type=None):
    """
    Uploads the given file to the AWS S3 bucket and key specified.
    :param aws_access_key_id: First part of AWS access key.
    :param aws_secret_access_key: Second part of AWS access key.
    :param file_to_s3: File object to be uploaded.
    :param bucket: S3 bucket name as string.
    :param key: Name attribute of the file object to be uploaded.
    :param callback: Function accepts two integer parameters, the first representing the number of bytes that have been
    successfully transmitted to S3 and the second representing the size of the to be transmitted object. Returns
    boolean indicating success/failure of upload.
    :param md5: MD5 checksum value to verify the integrity of the object.
    :param reduced_redundancy: S3 option that enables customers to reduce their costs
    by storing noncritical, reproducible data at lower levels of redundancy than S3's standard storage.
    :param content_type: Set the type of content in file object.
    :return: Boolean indicating success of upload.
    """
    try:
        size = os.fstat(file_to_s3.fileno()).st_size
    except:
        # Not all file objects implement fileno(), so we fall back on this
        file_to_s3.seek(0, os.SEEK_END)
        size = file_to_s3.tell()

    conn = boto.connect_s3(aws_access_key_id, aws_secret_access_key)
    bucket = conn.get_bucket(bucket, validate=True)
    k = Key(bucket)
    k.key = key
    if content_type:
        k.set_metadata('Content-Type', content_type)
    sent = k.set_contents_from_file(file_to_s3, cb=callback, md5=md5,
                                    reduced_redundancy=reduced_redundancy, rewind=True)

    # Rewind for later use
    file_to_s3.seek(0)

    if sent == size:
        return True
    return False

def join_dataframes():
    """
    Joining the dataframes of existing descriptor files from their urls into a single dataframe.
    :return: Dataframe after join over key column.
    """

    url_list = ['https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_constitution.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_con.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_kappa.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_estate.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_basak.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_property.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_charge.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_moe.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_burden.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_geary.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_moran.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_topology.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_geometric.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_cpsa.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_rdf.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_morse.csv',
                'https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/data/df_whim.csv'
                ]

    url_exist_list = []
    for url in url_list:
        try:
            r = urllib2.urlopen(url)
        except urllib2.URLError as e:
            r = e
        if r.code < 400:
            url_exist_list.append(url)

    i = 0
    df = [0] * len(url_exist_list)
    for url in url_exist_list:
        df[i] = pd.read_csv(url)

        df[i].drop(df[i].columns[0], axis=1, inplace=True)
        df[i].reset_index(drop=True, inplace=True)
        i += 1

    joined_df = df[0]
    for i in df[1:]:
        joined_df = joined_df.join(i)
    return joined_df

def choose_features(x_train, y_train, x_test, y_test, column_names, n_features):
    """
    Selecting the features of high importance to reduce feature space.
    :param x_train: Training set of features.
    :param x_test: Test set of features.
    :param y_train: Training target values
    :param y_test: Test target values.
    :param column_names: Names of columns in x
    :return desired_x_train: Reduced training set of features.
    :return desired_x_test: Reduced test set of features.
    """

    # Random forest feature importance
    choice = int(input("Type your choice of feature selection algorithm to be run:" + "\n" +
                       "1 for Random Forest Regression" + "\n" +
                       "2 for Randomized Lasso" + "\n"))
    if choice == 1:
        clf = RandomForestRegressor(n_jobs=-1, random_state=1, max_features=100)
        clf.fit(x_train, y_train.ravel())
        feature_importance = clf.feature_importances_
        feature_scores = pd.DataFrame({'feature': column_names, 'scores': feature_importance
                                       }).sort_values(by=['scores'], ascending=False)['feature'].tolist()
        selected_features = feature_scores[:n_features]
        x_train = pd.DataFrame(x_train, columns=column_names)
        desired_x_train = x_train[selected_features]
        x_test = pd.DataFrame(x_test, columns=column_names)
        desired_x_test = x_test[selected_features]
        y_train = pd.DataFrame(y_train, columns=['Activity_Score'])
        y_test = pd.DataFrame(y_test, columns=['Activity_Score'])

        x_train.to_csv('../data/x_train_postprocessing_rfr_%d.csv' % n_features)
        y_train.to_csv('../data/y_train_postprocessing_rfr_%d.csv' % n_features)
        x_test.to_csv('../data/x_test_postprocessing_rfr_%d.csv' % n_features)
        y_test.to_csv('../data/y_test_postprocessing_rfr_%d.csv' % n_features)

        return desired_x_train, y_train, desired_x_test, y_test

    elif choice == 2:
        clf = RandomizedLasso(alpha=0.025)
        clf.fit(x_train, y_train.ravel())
        feature_importance = clf.scores_
        feature_scores = pd.DataFrame({'feature': column_names, 'scores': feature_importance
                                       }).sort_values(by=['scores'], ascending=False)['feature'].tolist()
        selected_features = feature_scores[:n_features]
        x_train = pd.DataFrame(x_train, columns=column_names)
        desired_x_train = x_train[selected_features]
        x_test = pd.DataFrame(x_test, columns=column_names)
        desired_x_test = x_test[selected_features]
        y_train = pd.DataFrame(y_train, columns=['Activity_Score'])
        y_test = pd.DataFrame(y_test, columns=['Activity_Score'])

        x_train.to_csv('../data/x_train_postprocessing_rl_%d.csv' % n_features)
        y_train.to_csv('../data/y_train_postprocessing_rl_%d.csv' % n_features)
        x_test.to_csv('../data/x_test_postprocessing_rl_%d.csv' % n_features)
        y_test.to_csv('../data/y_test_postprocessing_rl_%d.csv' % n_features)

        return desired_x_train, y_train, desired_x_test, y_test
    else:
        print("Choose from options only")

def change_nan_infinite(dataframe):
    """
    Replacing NaN and infinite values from the dataframe with zeros.
    :param dataframe: Dataframe containing NaN and infinite values.
    :return dataframe: Dataframe with no NaN or infinite values.
    """

    dataframe.replace([np.inf, -np.inf], np.nan, inplace=True)
    dataframe.fillna(0, inplace=True)

    return dataframe

def plot_features(df_x, df_y):
    """
    Plotting each feature x and its corresponding value of target function y.
    :param df_x: Dataframe containing feature space.
    :param df_y: Dataframe containing target/output.
    """
    for column in df_x:
        plt.scatter(df_x[column], df_y)
        plt.title('%s effect on inhibition activity score trend' % df_x[column].name)
        plt.xlabel('%s' % df_x[column].name)
        plt.ylabel('Activity score')
        plt.savefig('../plots/feature_plots/%s.png' % df_x[column].name, bbox_inches='tight')

    return
<|endoftext|>


<commit_before># Copyright 2016 Goekcen Eraslan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os, json

from .network import autoencoder
from . import io

import numpy as np
from keras.optimizers import SGD
from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from keras import backend as K

def train(X, hidden_size=32, l2_coef=0., optimizer=None, learning_rate=0.01,
          log_dir='logs', aetype=None, epochs=200, reduce_lr_epoch=20,
          early_stopping_epoch=40, batch_size=32,
          censortype=None, censorthreshold=None,
          hyperpar=None, **kwargs):

    model, _, _, loss = autoencoder(X['shape'][1], hidden_size=hidden_size,
                                    l2_coef=l2_coef, aetype=aetype)
    if optimizer is None:
        optimizer = SGD(lr=learning_rate, momentum=0.9, nesterov=True)

    model.compile(loss=loss, optimizer=optimizer)

    # Callbacks
    checkpointer = ModelCheckpoint(filepath="%s/weights.hdf5" % log_dir,
                                   verbose=1)
    es_cb = EarlyStopping(monitor='val_loss', min_delta=0,
                          patience=early_stopping_epoch,
                          verbose=0, mode='auto')

    lr_cb = ReduceLROnPlateau(monitor='val_loss', factor=0.1,
                              patience=reduce_lr_epoch,
                              verbose=0, mode='auto', epsilon=0.0001,
                              cooldown=0, min_lr=0)
    tb_cb = TensorBoard(log_dir=log_dir, histogram_freq=1)
    callbacks = []

    if reduce_lr_epoch: callbacks.append(lr_cb)
    if early_stopping_epoch: callbacks.append(es_cb)

    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    with open('%s/arch.json' % log_dir, 'w') as f:
        json.dump(model.to_json(), f)

    print(model.summary())

    fold_losses = list()
    for data in X['folds']:
        loss = model.fit(data['train'], data['train'],
                         nb_epoch=epochs,
                         batch_size=batch_size,
                         shuffle=True,
                         callbacks=callbacks,
                         validation_data=(data['val'], data['val']),
                         **kwargs)
        #model.evaluate(data['test'], data['test'], batch_size=32,
        #               verbose=1, sample_weight=None)
        fold_losses.append(loss.history)

    # run final training on full dataset
    full_data = np.concatenate((X['folds'][0]['train'],
                                X['folds'][0]['val'],
                                X['folds'][0]['test']))

    loss = model.fit(full_data, full_data,
                     nb_epoch=epochs,
                     batch_size=batch_size,
                     shuffle=True,
                     callbacks=[tb_cb, checkpointer],
                     **kwargs)

    #https://github.com/tensorflow/tensorflow/issues/3388
    #K.clear_session()

    return model, {'full': loss.history, 'fold': fold_losses}

def train_with_args(args):
    X = io.read_from_file(args.trainingset)

    train(X=X, hidden_size=args.hiddensize,
          learning_rate=args.learningrate,
          log_dir=args.logdir,
          aetype=args.type,
          epochs=args.epochs,
          censorthreshold=args.censorthreshold,
          censortype=args.censortype,
          hyperpar=args.hyperpar)
<commit_msg>Monitor training loss on full data fit<commit_after># Copyright 2016 Goekcen Eraslan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os, json

from .network import autoencoder
from . import io

import numpy as np
from keras.optimizers import SGD
from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from keras import backend as K

def train(X, hidden_size=32, l2_coef=0., optimizer=None, learning_rate=0.01,
          log_dir='logs', aetype=None, epochs=200, reduce_lr_epoch=20,
          early_stopping_epoch=40, batch_size=32,
          censortype=None, censorthreshold=None,
          hyperpar=None, **kwargs):

    model, _, _, loss = autoencoder(X['shape'][1], hidden_size=hidden_size,
                                    l2_coef=l2_coef, aetype=aetype)
    if optimizer is None:
        optimizer = SGD(lr=learning_rate, momentum=0.9, nesterov=True)

    model.compile(loss=loss, optimizer=optimizer)

    # Callbacks
    checkpointer = ModelCheckpoint(filepath="%s/weights.hdf5" % log_dir,
                                   verbose=1)
    es_cb = EarlyStopping(monitor='val_loss', patience=early_stopping_epoch)
    es_cb_train = EarlyStopping(monitor='train_loss', patience=early_stopping_epoch)

    lr_cb = ReduceLROnPlateau(monitor='val_loss', patience=reduce_lr_epoch)
    lr_cb_train = ReduceLROnPlateau(monitor='train_loss', patience=reduce_lr_epoch)

    tb_cb = TensorBoard(log_dir=log_dir, histogram_freq=1)
    callbacks = []
    callbacks_train = []

    if reduce_lr_epoch:
        callbacks.append(lr_cb)
        callbacks_train.append(lr_cb_train)
    if early_stopping_epoch:
        callbacks.append(es_cb)
        callbacks_train.append(es_cb_train)

    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    with open('%s/arch.json' % log_dir, 'w') as f:
        json.dump(model.to_json(), f)

    print(model.summary())

    fold_losses = list()
    for data in X['folds']:
        loss = model.fit(data['train'], data['train'],
                         nb_epoch=epochs,
                         batch_size=batch_size,
                         shuffle=True,
                         callbacks=callbacks,
                         validation_data=(data['val'], data['val']),
                         **kwargs)
        #model.evaluate(data['test'], data['test'], batch_size=32,
        #               verbose=1, sample_weight=None)
        fold_losses.append(loss.history)

    # run final training on full dataset
    full_data = np.concatenate((X['folds'][0]['train'],
                                X['folds'][0]['val'],
                                X['folds'][0]['test']))

    loss = model.fit(full_data, full_data,
                     nb_epoch=epochs,
                     batch_size=batch_size,
                     shuffle=True,
                     callbacks=callbacks_train.extend([tb_cb, checkpointer]),
                     **kwargs)

    #https://github.com/tensorflow/tensorflow/issues/3388
    #K.clear_session()

    return model, {'full': loss.history, 'fold': fold_losses}

def train_with_args(args):
    X = io.read_from_file(args.trainingset)

    train(X=X, hidden_size=args.hiddensize,
          learning_rate=args.learningrate,
          log_dir=args.logdir,
          aetype=args.type,
          epochs=args.epochs,
          censorthreshold=args.censorthreshold,
          censortype=args.censortype,
          hyperpar=args.hyperpar)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import h5py

infiles = sys.argv[1:]

for infile in infiles:
    with h5py.File(infile, 'a') as h5:
        print(infile)
        if 'matrix' in h5 and not 'pixels' in h5:
            print('renaming matrix --> pixels')
            h5['pixels'] = h5['matrix']

        if 'scaffolds' in h5 and not 'chroms' in h5:
            print('renaming scaffolds --> chroms')
            h5['chroms'] = h5['scaffolds']

<commit_msg>Change format version metadata in v0to1<commit_after>#!/usr/bin/env python
import sys
import h5py

infiles = sys.argv[1:]

for infile in infiles:
    with h5py.File(infile, 'a') as h5:
        print(infile)
        if 'format-version' in h5.attrs and h5.attrs['format-version'] < 1:

            if 'matrix' in h5 and not 'pixels' in h5:
                print('renaming matrix --> pixels')
                h5['pixels'] = h5['matrix']

            if 'scaffolds' in h5 and not 'chroms' in h5:
                print('renaming scaffolds --> chroms')
                h5['chroms'] = h5['scaffolds']

            h5.attrs['format-version'] = 1

<|endoftext|>


<commit_before>#!/usr/bin/env python

import time
import sys
import os
import threading
import socket
import fcntl
import struct
try:
    import queue
except:
    import Queue as queue

import argparse

from ctypes import *

from operator import itemgetter

MAX_CAMERAS = 64

class TcamCamera(Structure):
    _fields_ = [("model_name", c_char * 64),
                ("serial_number", c_char * 64),
                ("current_ip", c_char * 16),
                ("current_gateway", c_char * 16),
                ("current_netmask", c_char * 16),
                ("persistent_ip", c_char * 16),
                ("persistent_gateway", c_char * 16),
                ("persistent_netmask", c_char * 16),
                ("user_defined_name", c_char * 64),
                ("firmware_version", c_char * 64),
                ("mac_address", c_char * 64),
                ("interface_name", c_char * 64),
                ("is_static_ip", c_int),
                ("is_dhcp_enabled", c_int),
                ("is_reachable", c_int),
                ("is_controllable", c_int)
    ]

DISCOVER_CALLBACK_FUNC = CFUNCTYPE(None, TcamCamera)
UPLOAD_CALLBACK_FUNC = CFUNCTYPE(None, c_char_p, c_int)

class CameraController:
    def __init__(self):
        _path = os.path.dirname(__file__)
        if not _path:
            _path = "."
        self.dll = cdll.LoadLibrary(os.path.join(_path,"gigewrapper.so"))
        self.dll.init()
        self.dll.set_persistent_parameter_s.argtypes = [c_char_p, c_char_p, c_char_p]
        self.dll.set_persistent_parameter_i.argtypes = [c_char_p, c_char_p, c_int]
        self.dll.rescue.argtypes = [c_char_p, c_char_p, c_char_p, c_char_p]
        self.dll.upload_firmware.argtypes = [c_char_p, c_char_p, UPLOAD_CALLBACK_FUNC]
        self.cameras = []

    SUCCESS = 0x0
    FAILURE = 0x8000
    NO_DEVICE = 0x8001
    INVALID_PARAMETER = 0x8002

    @staticmethod
    def __getdict(struct):
        d = dict((field, getattr(struct, field)) for field, _ in struct._fields_)
        for f in d:
            if type(d[f]) == bytes:
                d[f] = d[f].decode("utf-8")
        return d

    def __discover_callback(self, camera):
        self.cameras.append(self.__getdict(camera))

    def discover(self, get_persistent_values=False):
        self.cameras = []
        return self.dll.get_camera_list(DISCOVER_CALLBACK_FUNC(self.__discover_callback), get_persistent_values)

    def set_persistent_parameter(self, identifier, key, value):
        if type(value) == str:
            return self.dll.set_persistent_parameter_s(bytes(identifier, "utf-8"),
                                                       bytes(key, "utf-8"),
                                                       bytes(value, "utf-8"))
        else:
            return self.dll.set_persistent_parameter_i(bytes(identifier, "utf-8"),
                                                       bytes(key, "utf-8"), value)

    def upload_firmware(self, identifier, _path, callback):
        return self.dll.upload_firmware(bytes(identifier, "utf-8"), bytes(_path, "utf-8"), callback)

    def get_camera_details(self, identifier):
        cam = TcamCamera()
        self.dll.get_camera_details(bytes(identifier, "utf-8"), byref(cam))
        return self.__getdict(cam)

    def rescue(self, identifier, ip, netmask, gateway):
        mac = None
        for cam in self.cameras:
            if identifier in [cam["serial_number"], cam["user_defined_name"], cam["mac_address"]]:
                if mac is not None:
                    print("Camera identifier is ambiguous")
                    return -2
                mac = cam["mac_address"]

        if mac is None:
            print("No matching camera")
            return -1

        # print("send rescue to: ", mac, ip, netmask, gateway)

        self.dll.rescue(bytes(mac, "utf-8"), bytes(ip, "utf-8"), bytes(netmask, "utf-8"), bytes(gateway, "utf-8"))

def print_usage():
    print("""Usage:
{0} list
    :: Display a list of all available cameras

{0} rescue CAMERA_IDENTIFIER ip=IPADRESS netmask=NETMASK gateway=GATEWAY
    :: Temporarily set the IP address configuration of the camera

{0} set CAMERA_IDENTIFIER [ip=IPADDRESS netmask=NETMASK gateway=GATEWAY static=True | dhcp=True]
    :: Store permanent IP configuration settings

{0} upload CAMERA_IDENTIFIER FIRMWARE_FILE
    :: Upload a firmware file to a camera

{0} batchupload INTERFACE_NAME FIRMWARE_FILE
    :: Auto-configure the IP-Adresses of all cameras detected on the given interface to
       the IP addresses in the subnet, starting with the IP address x.x.x.10 .
       Then start up to 8 threads simultaneously to upload the given firmware file.abs

CAMERA_IDENTIFIER: Can be either the serial number, the user defined name or the MAC address
""".format(sys.argv[0]))

def list():
    ctrl = CameraController()
    ctrl.discover()
    lst = sorted(ctrl.cameras, key=itemgetter("model_name"))
    lst = sorted(lst, key=itemgetter("serial_number"))
    lst = sorted(lst, key=itemgetter("user_defined_name"))
    lst = sorted(lst, key=itemgetter("interface_name"))

    for cam in lst:
        s = (cam["model_name"].ljust(20) +
             cam["serial_number"].ljust(10) +
             cam["user_defined_name"].ljust(20) +
             cam["current_ip"].ljust(17) +
             cam["interface_name"].ljust(12))
        print(s)

def rescue():
    ctrl = CameraController()
    ctrl.discover()

    identifier = sys.argv[2]
    args = sys.argv[3:]
    ip = None
    netmask = None
    gateway = None
    for arg in args:
        key, value = arg.split("=")
        if key == "ip":
            ip = value
        elif key == "netmask":
            netmask = value
        elif key == "gateway":
            gateway = value

    if ip is None or netmask is None or gateway is None:
        print("Argument error")
        return -2

    return ctrl.rescue(identifier, ip, netmask, gateway)

class FirmwareUploadCallback:
    def __init__(self):
        pass

    def func(self, msg, progress):
        msg = msg.decode("utf-8")
        sys.stdout.write("\r%s%% %s" % (str(progress).rjust(3), msg))

def upload():
    ctrl = CameraController()
    ctrl.discover()

    identifier = sys.argv[2]
    _path = os.path.abspath(os.path.realpath(sys.argv[3]))
    fwupcb = FirmwareUploadCallback()
    res = ctrl.upload_firmware(identifier, _path, UPLOAD_CALLBACK_FUNC(fwupcb.func))

    if res != 0:
        print ("Upload failed, result: %d" % (res,))

class BatchUploadThread(threading.Thread):
    def __init__(self, workqueue):
        threading.Thread.__init__(self)
        self.progress = 0
        self.workqueue = workqueue
        self.results = []

    def func(self, msg, progress):
        msg = msg.decode("utf-8")
        self.progress = progress

    def run(self):
        while not self.workqueue.empty():
            self.progress = 0
            try:
                camctrl, cam, fwpath = self.workqueue.get_nowait()
            except queue.QueueEmpty:
                break
            result = camctrl.upload_firmware(cam["serial_number"], fwpath, UPLOAD_CALLBACK_FUNC(self.func))
            self.results.append((cam, result, self.progress))
            self.workqueue.task_done()
        self.progress = None

def get_ip_address(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        0x8915,  # SIOCGIFADDR
        struct.pack('256s', bytes(ifname[:15], "utf-8"))
    )[20:24])

def batchrescue(ctrl, cameras, baseip):
    ip_a, ip_b, ip_c, ip_d = baseip
    for cam in cameras:
        ip = "%d.%d.%d.%d" % (ip_a, ip_b, ip_c, ip_d)
        netmask = "255.255.255.0"
        gateway = "0.0.0.0"
        ctrl.rescue(cam["serial_number"], ip, netmask, gateway)
        ip_d += 1

def batchupload():
    ctrl = CameraController()
    ctrl.discover()

    iface = sys.argv[2]
    _path = os.path.abspath(os.path.realpath(sys.argv[3]))

    cameras = []
    for cam in ctrl.cameras:
        if cam["interface_name"] == iface:
            cameras.append(cam)

    if not cameras:
        print("No cameras found to update")
        return

    ifaceip = get_ip_address(iface)
    a,b,c,d = [int(x) for x in ifaceip.split(".")]
    d = 10
    print("Configuring cameras for address range: {0}.{1}.{2}.{3} .. {0}.{1}.{2}.{4}".format(a,b,c,d,d+len(cameras)-1))
    batchrescue(ctrl, cameras, (a,b,c,d))
    time.sleep(1)

    print("Rediscovering cameras")
    del ctrl
    ctrl = CameraController()
    ctrl.discover()
    cameras = []
    for cam in ctrl.cameras:
        if cam["interface_name"] == iface:
            cameras.append(cam)

    print("Starting Threads...")
    threads = []
    workqueue = queue.Queue()
    for cam in cameras:
        workqueue.put((ctrl, cam, _path))

    numthreads = min(workqueue.qsize(), 8)
    for t in range(numthreads):
        t = BatchUploadThread(workqueue)
        t.start()
        threads.append(t)

    done = False
    while not done:
        time.sleep(0.5)
        wc = threading.active_count() -1
        remain = workqueue.qsize()
        s = "Working:" + str(wc).ljust(3) + " Remaining:" + str(remain).ljust(3) + " "
        for t in threads:
            if t.progress is None:
                progress = "---".center(6)
            else:
                progress = "%d%%" % (t.progress)
            s += progress.ljust(6)
        sys.stdout.write("\r%s" % (s))
        done = threading.active_count() == 1

    print("\r".ljust(79))

    results = []
    for t in threads:
        results += t.results

    print("Finished. Update summary:")
    failed = []
    for result in results:
        if result[1] == 0:
            restxt = "Update completed successfully"
        else:
            restxt = "Firmware update FAILED! Upload will be retried (%d)." % (result[1])
            failed.append(result[0])
        print("Device: %s, completed: %d%%, result: %s" % (result[0]["serial_number"], result[2], restxt))

    if failed:
        print("At least one device failed to update")
        print("Retrying the upload in sequential mode.")
        del ctrl
        ctrl = CameraController()
        ctrl.discover()
        time.sleep(1)
        for cam in failed:
            print("Uploading to device: %s" % (cam["serial_number"]))
            fwupcb = FirmwareUploadCallback()
            res = ctrl.upload_firmware(cam["serial_number"], _path, UPLOAD_CALLBACK_FUNC(fwupcb.func))

            if res != 0:
                print ("Upload failed, result: %d" % (res,))

def _parsebool(value):
    TRUES = ["true", "on", "1", "yes"]
    if value.lower() in TRUES:
        return 1
    return 0

def set_persistent():
    BOOLARGS = ["dhcp", "static"]
    EXCLUSIVE = ["dhcp", "static"]
    identifier = sys.argv[2]
    args = sys.argv[3:]

    ctrl = CameraController()
    ctrl.discover()

    for arg in args:
        note = ""
        sys.stdout.write(arg)
        key, value = arg.split("=")
        if key in BOOLARGS:
            value = _parsebool(value)

        if key in EXCLUSIVE and value:
            for excl in EXCLUSIVE:
                if excl != key:
                    args.append("%s=false" % (excl))
                    note = "(disabling '%s' because '%s' is enabled)" % (excl, key)

        res = ctrl.set_persistent_parameter(identifier, key, value)
        if res != 0:
            print(" -> FAILED")
            break
        else:
            print(" -> OK " + note)
    if res != 0:
        print("Failed to set parameter '%s'" % (key))

def show_info():
    ctrl = CameraController()
    ctrl.discover()

    def bool_to_txt(val):
        if val:
            return "enabled"
        else:
            return "disabled"

    identifier = sys.argv[2]
    cam = ctrl.get_camera_details(identifier)

    print("Model:".ljust(16) + cam["model_name"])
    print("Serial:".ljust(16) + cam["serial_number"])
    print("Firmware:".ljust(16) + cam["firmware_version"])
    print("UserName:".ljust(16) + cam["user_defined_name"])
    print()
    print("MAC Address:".ljust(30) + cam["mac_address"])
    print("Current IP:".ljust(30) + cam["current_ip"])
    print("Current Netmask:".ljust(30) + cam["current_netmask"])
    print("Current Gateway:".ljust(30) + cam["current_gateway"])
    print("")
    print("DHCP is: ".ljust(16) + bool_to_txt(cam["is_dhcp_enabled"]))
    print("Static is: ".ljust(16) + bool_to_txt(cam["is_static_ip"]))
    print("")
    print("Persistent IP:".ljust(30) + cam["persistent_ip"])
    print("Persistent Netmask:".ljust(30) + cam["persistent_netmask"])
    print("Persistent Gateway:".ljust(30) + cam["persistent_gateway"])

if __name__ == "__main__":
    # parser = argparse.ArgumentParser(description="The Imaging Source Gigabit Ethernet camera configuration tool")
    # parser.add_argument("command", nargs="+", help="command to execute")
    # parser.add_argument("identifier", nargs="?", help="")

    if len(sys.argv) < 2:
        print_usage()
        sys.exit(1)

    if sys.argv[1] == "list":
        list()
    elif sys.argv[1] == "rescue":
        rescue()
    elif sys.argv[1] == "upload":
        upload()
    elif sys.argv[1] == "batchupload":
        batchupload()
    elif sys.argv[1] == "set":
        set_persistent()
    elif sys.argv[1] == "info":
        show_info()
    else:
        print_usage()

    # camctl = CameraController()
    # camctl.discover()
    # print(camctl.cameras)
    # #print(hex(camctl.set_persistent_parameter(b"11719912", b"foo", b"bar")))
    # fwupcb = FirmwareUploadCallback()
    # camctl.upload_firmware(b"11719912", b"GigE3Firmware.1672-All.fwpack", UPLOAD_CALLBACK_FUNC(fwupcb.func))
<commit_msg>gigetool: Require python3 for now<commit_after>#!/usr/bin/env python3

import time
import sys
import os
import threading
import socket
import fcntl
import struct
try:
    import queue
except:
    import Queue as queue

import argparse

from ctypes import *

from operator import itemgetter

MAX_CAMERAS = 64

class TcamCamera(Structure):
    _fields_ = [("model_name", c_char * 64),
                ("serial_number", c_char * 64),
                ("current_ip", c_char * 16),
                ("current_gateway", c_char * 16),
                ("current_netmask", c_char * 16),
                ("persistent_ip", c_char * 16),
                ("persistent_gateway", c_char * 16),
                ("persistent_netmask", c_char * 16),
                ("user_defined_name", c_char * 64),
                ("firmware_version", c_char * 64),
                ("mac_address", c_char * 64),
                ("interface_name", c_char * 64),
                ("is_static_ip", c_int),
                ("is_dhcp_enabled", c_int),
                ("is_reachable", c_int),
                ("is_controllable", c_int)
    ]

DISCOVER_CALLBACK_FUNC = CFUNCTYPE(None, TcamCamera)
UPLOAD_CALLBACK_FUNC = CFUNCTYPE(None, c_char_p, c_int)

class CameraController:
    def __init__(self):
        _path = os.path.dirname(__file__)
        if not _path:
            _path = "."
        self.dll = cdll.LoadLibrary(os.path.join(_path,"gigewrapper.so"))
        self.dll.init()
        self.dll.set_persistent_parameter_s.argtypes = [c_char_p, c_char_p, c_char_p]
        self.dll.set_persistent_parameter_i.argtypes = [c_char_p, c_char_p, c_int]
        self.dll.rescue.argtypes = [c_char_p, c_char_p, c_char_p, c_char_p]
        self.dll.upload_firmware.argtypes = [c_char_p, c_char_p, UPLOAD_CALLBACK_FUNC]
        self.cameras = []

    SUCCESS = 0x0
    FAILURE = 0x8000
    NO_DEVICE = 0x8001
    INVALID_PARAMETER = 0x8002

    @staticmethod
    def __getdict(struct):
        d = dict((field, getattr(struct, field)) for field, _ in struct._fields_)
        for f in d:
            if type(d[f]) == bytes:
                d[f] = d[f].decode("utf-8")
        return d

    def __discover_callback(self, camera):
        self.cameras.append(self.__getdict(camera))

    def discover(self, get_persistent_values=False):
        self.cameras = []
        return self.dll.get_camera_list(DISCOVER_CALLBACK_FUNC(self.__discover_callback), get_persistent_values)

    def set_persistent_parameter(self, identifier, key, value):
        if type(value) == str:
            return self.dll.set_persistent_parameter_s(bytes(identifier, "utf-8"),
                                                       bytes(key, "utf-8"),
                                                       bytes(value, "utf-8"))
        else:
            return self.dll.set_persistent_parameter_i(bytes(identifier, "utf-8"),
                                                       bytes(key, "utf-8"), value)

    def upload_firmware(self, identifier, _path, callback):
        return self.dll.upload_firmware(bytes(identifier, "utf-8"), bytes(_path, "utf-8"), callback)

    def get_camera_details(self, identifier):
        cam = TcamCamera()
        self.dll.get_camera_details(bytes(identifier, "utf-8"), byref(cam))
        return self.__getdict(cam)

    def rescue(self, identifier, ip, netmask, gateway):
        mac = None
        for cam in self.cameras:
            if identifier in [cam["serial_number"], cam["user_defined_name"], cam["mac_address"]]:
                if mac is not None:
                    print("Camera identifier is ambiguous")
                    return -2
                mac = cam["mac_address"]

        if mac is None:
            print("No matching camera")
            return -1

        # print("send rescue to: ", mac, ip, netmask, gateway)

        self.dll.rescue(bytes(mac, "utf-8"), bytes(ip, "utf-8"), bytes(netmask, "utf-8"), bytes(gateway, "utf-8"))

def print_usage():
    print("""Usage:
{0} list
    :: Display a list of all available cameras

{0} rescue CAMERA_IDENTIFIER ip=IPADRESS netmask=NETMASK gateway=GATEWAY
    :: Temporarily set the IP address configuration of the camera

{0} set CAMERA_IDENTIFIER [ip=IPADDRESS netmask=NETMASK gateway=GATEWAY static=True | dhcp=True]
    :: Store permanent IP configuration settings

{0} upload CAMERA_IDENTIFIER FIRMWARE_FILE
    :: Upload a firmware file to a camera

{0} batchupload INTERFACE_NAME FIRMWARE_FILE
    :: Auto-configure the IP-Adresses of all cameras detected on the given interface to
       the IP addresses in the subnet, starting with the IP address x.x.x.10 .
       Then start up to 8 threads simultaneously to upload the given firmware file.abs

CAMERA_IDENTIFIER: Can be either the serial number, the user defined name or the MAC address
""".format(sys.argv[0]))

def list():
    ctrl = CameraController()
    ctrl.discover()
    lst = sorted(ctrl.cameras, key=itemgetter("model_name"))
    lst = sorted(lst, key=itemgetter("serial_number"))
    lst = sorted(lst, key=itemgetter("user_defined_name"))
    lst = sorted(lst, key=itemgetter("interface_name"))

    for cam in lst:
        s = (cam["model_name"].ljust(20) +
             cam["serial_number"].ljust(10) +
             cam["user_defined_name"].ljust(20) +
             cam["current_ip"].ljust(17) +
             cam["interface_name"].ljust(12))
        print(s)

def rescue():
    ctrl = CameraController()
    ctrl.discover()

    identifier = sys.argv[2]
    args = sys.argv[3:]
    ip = None
    netmask = None
    gateway = None
    for arg in args:
        key, value = arg.split("=")
        if key == "ip":
            ip = value
        elif key == "netmask":
            netmask = value
        elif key == "gateway":
            gateway = value

    if ip is None or netmask is None or gateway is None:
        print("Argument error")
        return -2

    return ctrl.rescue(identifier, ip, netmask, gateway)

class FirmwareUploadCallback:
    def __init__(self):
        pass

    def func(self, msg, progress):
        msg = msg.decode("utf-8")
        sys.stdout.write("\r%s%% %s" % (str(progress).rjust(3), msg))

def upload():
    ctrl = CameraController()
    ctrl.discover()

    identifier = sys.argv[2]
    _path = os.path.abspath(os.path.realpath(sys.argv[3]))
    fwupcb = FirmwareUploadCallback()
    res = ctrl.upload_firmware(identifier, _path, UPLOAD_CALLBACK_FUNC(fwupcb.func))

    if res != 0:
        print ("Upload failed, result: %d" % (res,))

class BatchUploadThread(threading.Thread):
    def __init__(self, workqueue):
        threading.Thread.__init__(self)
        self.progress = 0
        self.workqueue = workqueue
        self.results = []

    def func(self, msg, progress):
        msg = msg.decode("utf-8")
        self.progress = progress

    def run(self):
        while not self.workqueue.empty():
            self.progress = 0
            try:
                camctrl, cam, fwpath = self.workqueue.get_nowait()
            except queue.QueueEmpty:
                break
            result = camctrl.upload_firmware(cam["serial_number"], fwpath, UPLOAD_CALLBACK_FUNC(self.func))
            self.results.append((cam, result, self.progress))
            self.workqueue.task_done()
        self.progress = None

def get_ip_address(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        0x8915,  # SIOCGIFADDR
        struct.pack('256s', bytes(ifname[:15], "utf-8"))
    )[20:24])

def batchrescue(ctrl, cameras, baseip):
    ip_a, ip_b, ip_c, ip_d = baseip
    for cam in cameras:
        ip = "%d.%d.%d.%d" % (ip_a, ip_b, ip_c, ip_d)
        netmask = "255.255.255.0"
        gateway = "0.0.0.0"
        ctrl.rescue(cam["serial_number"], ip, netmask, gateway)
        ip_d += 1

def batchupload():
    ctrl = CameraController()
    ctrl.discover()

    iface = sys.argv[2]
    _path = os.path.abspath(os.path.realpath(sys.argv[3]))

    cameras = []
    for cam in ctrl.cameras:
        if cam["interface_name"] == iface:
            cameras.append(cam)

    if not cameras:
        print("No cameras found to update")
        return

    ifaceip = get_ip_address(iface)
    a,b,c,d = [int(x) for x in ifaceip.split(".")]
    d = 10
    print("Configuring cameras for address range: {0}.{1}.{2}.{3} .. {0}.{1}.{2}.{4}".format(a,b,c,d,d+len(cameras)-1))
    batchrescue(ctrl, cameras, (a,b,c,d))
    time.sleep(1)

    print("Rediscovering cameras")
    del ctrl
    ctrl = CameraController()
    ctrl.discover()
    cameras = []
    for cam in ctrl.cameras:
        if cam["interface_name"] == iface:
            cameras.append(cam)

    print("Starting Threads...")
    threads = []
    workqueue = queue.Queue()
    for cam in cameras:
        workqueue.put((ctrl, cam, _path))

    numthreads = min(workqueue.qsize(), 8)
    for t in range(numthreads):
        t = BatchUploadThread(workqueue)
        t.start()
        threads.append(t)

    done = False
    while not done:
        time.sleep(0.5)
        wc = threading.active_count() -1
        remain = workqueue.qsize()
        s = "Working:" + str(wc).ljust(3) + " Remaining:" + str(remain).ljust(3) + " "
        for t in threads:
            if t.progress is None:
                progress = "---".center(6)
            else:
                progress = "%d%%" % (t.progress)
            s += progress.ljust(6)
        sys.stdout.write("\r%s" % (s))
        done = threading.active_count() == 1

    print("\r".ljust(79))

    results = []
    for t in threads:
        results += t.results

    print("Finished. Update summary:")
    failed = []
    for result in results:
        if result[1] == 0:
            restxt = "Update completed successfully"
        else:
            restxt = "Firmware update FAILED! Upload will be retried (%d)." % (result[1])
            failed.append(result[0])
        print("Device: %s, completed: %d%%, result: %s" % (result[0]["serial_number"], result[2], restxt))

    if failed:
        print("At least one device failed to update")
        print("Retrying the upload in sequential mode.")
        del ctrl
        ctrl = CameraController()
        ctrl.discover()
        time.sleep(1)
        for cam in failed:
            print("Uploading to device: %s" % (cam["serial_number"]))
            fwupcb = FirmwareUploadCallback()
            res = ctrl.upload_firmware(cam["serial_number"], _path, UPLOAD_CALLBACK_FUNC(fwupcb.func))

            if res != 0:
                print ("Upload failed, result: %d" % (res,))

def _parsebool(value):
    TRUES = ["true", "on", "1", "yes"]
    if value.lower() in TRUES:
        return 1
    return 0

def set_persistent():
    BOOLARGS = ["dhcp", "static"]
    EXCLUSIVE = ["dhcp", "static"]
    identifier = sys.argv[2]
    args = sys.argv[3:]

    ctrl = CameraController()
    ctrl.discover()

    for arg in args:
        note = ""
        sys.stdout.write(arg)
        key, value = arg.split("=")
        if key in BOOLARGS:
            value = _parsebool(value)

        if key in EXCLUSIVE and value:
            for excl in EXCLUSIVE:
                if excl != key:
                    args.append("%s=false" % (excl))
                    note = "(disabling '%s' because '%s' is enabled)" % (excl, key)

        res = ctrl.set_persistent_parameter(identifier, key, value)
        if res != 0:
            print(" -> FAILED")
            break
        else:
            print(" -> OK " + note)
    if res != 0:
        print("Failed to set parameter '%s'" % (key))

def show_info():
    ctrl = CameraController()
    ctrl.discover()

    def bool_to_txt(val):
        if val:
            return "enabled"
        else:
            return "disabled"

    identifier = sys.argv[2]
    cam = ctrl.get_camera_details(identifier)

    print("Model:".ljust(16) + cam["model_name"])
    print("Serial:".ljust(16) + cam["serial_number"])
    print("Firmware:".ljust(16) + cam["firmware_version"])
    print("UserName:".ljust(16) + cam["user_defined_name"])
    print()
    print("MAC Address:".ljust(30) + cam["mac_address"])
    print("Current IP:".ljust(30) + cam["current_ip"])
    print("Current Netmask:".ljust(30) + cam["current_netmask"])
    print("Current Gateway:".ljust(30) + cam["current_gateway"])
    print("")
    print("DHCP is: ".ljust(16) + bool_to_txt(cam["is_dhcp_enabled"]))
    print("Static is: ".ljust(16) + bool_to_txt(cam["is_static_ip"]))
    print("")
    print("Persistent IP:".ljust(30) + cam["persistent_ip"])
    print("Persistent Netmask:".ljust(30) + cam["persistent_netmask"])
    print("Persistent Gateway:".ljust(30) + cam["persistent_gateway"])

if __name__ == "__main__":
    # parser = argparse.ArgumentParser(description="The Imaging Source Gigabit Ethernet camera configuration tool")
    # parser.add_argument("command", nargs="+", help="command to execute")
    # parser.add_argument("identifier", nargs="?", help="")

    if len(sys.argv) < 2:
        print_usage()
        sys.exit(1)

    if sys.argv[1] == "list":
        list()
    elif sys.argv[1] == "rescue":
        rescue()
    elif sys.argv[1] == "upload":
        upload()
    elif sys.argv[1] == "batchupload":
        batchupload()
    elif sys.argv[1] == "set":
        set_persistent()
    elif sys.argv[1] == "info":
        show_info()
    else:
        print_usage()

    # camctl = CameraController()
    # camctl.discover()
    # print(camctl.cameras)
    # #print(hex(camctl.set_persistent_parameter(b"11719912", b"foo", b"bar")))
    # fwupcb = FirmwareUploadCallback()
    # camctl.upload_firmware(b"11719912", b"GigE3Firmware.1672-All.fwpack", UPLOAD_CALLBACK_FUNC(fwupcb.func))
<|endoftext|>


<commit_before># flake8: noqa
# There's no way to ignore "F401 '...' imported but unused" warnings in this
# module, but to preserve other warnings. So, don't check this module at all.

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.0.6"

from .constants import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    HUGGINGFACE_CO_URL_HOME,
    HUGGINGFACE_CO_URL_TEMPLATE,
    PYTORCH_WEIGHTS_NAME,
    REPO_TYPE_DATASET,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
)
from .file_download import cached_download, hf_hub_url
from .hf_api import HfApi, HfFolder
from .hub_mixin import ModelHubMixin
from .repository import Repository
<commit_msg>0.0.7<commit_after># flake8: noqa
# There's no way to ignore "F401 '...' imported but unused" warnings in this
# module, but to preserve other warnings. So, don't check this module at all.

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.0.7"

from .constants import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    HUGGINGFACE_CO_URL_HOME,
    HUGGINGFACE_CO_URL_TEMPLATE,
    PYTORCH_WEIGHTS_NAME,
    REPO_TYPE_DATASET,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
)
from .file_download import cached_download, hf_hub_url
from .hf_api import HfApi, HfFolder
from .hub_mixin import ModelHubMixin
from .repository import Repository
<|endoftext|>


<commit_before>"""
    tubeModel.py -
        Determines the steady state temperature of the hyperloop tube.
        Calculates Q released/absorbed by hyperloop tube due to:
        Internal Convection, Tube Conduction, Ambient Natural Convection, Solar Flux In, Radiation Out

    -original calculations from Jeff Berton, ported and extended by Jeff Chin

    Compatible with OpenMDAO v0.8.1
"""
from math import log, pi, sqrt, e

from openmdao.main.api import Assembly, Component
from openmdao.lib.drivers.api import BroydenSolver
from openmdao.lib.datatypes.api import Float, Bool
from openmdao.main.api import convert_units as cu

from pycycle.api import FlowStationVar

class TubeWallTemp(Component):
    """ Calculates Q released/absorbed by the hyperloop tube """
    #--Inputs--
    #Hyperloop Parameters/Design Variables
    diameter_outer_tube = Float(2.23, units = 'm', iotype='in', desc='tube outer diameter') #7.3ft
    length_tube = Float(482803, units = 'm', iotype='in', desc='Length of entire Hyperloop') #300 miles, 1584000ft
    num_pods = Float(34, units = 'K', iotype='in', desc='Number of Pods in the Tube at a given time') #
    temp_boundary = Float(322.0, units = 'K', iotype='in', desc='Average Temperature of the tube wall') #
    temp_outside_ambient = Float(305.6, units = 'K', iotype='in', desc='Average Temperature of the outside air') #
    nozzle_air = FlowStationVar(iotype="in", desc="air exiting the pod nozzle", copy=None)
    bearing_air = FlowStationVar(iotype="in", desc="air exiting the air bearings", copy=None)
    #constants
    solar_insolation = Float(1000., iotype="in", units = 'W/m**2', desc='solar irradiation at sea level on a clear day') #
    nn_incidence_factor = Float(0.7, iotype="in", desc='Non-normal incidence factor') #
    surface_reflectance = Float(0.5, desc='Solar Reflectance Index') #
    q_per_area_solar = Float(350., units = 'W/m**2', desc='Solar Heat Rate Absorbed per Area') #
    q_total_solar = Float(375989751., iotype="in", units = 'W', desc='Solar Heat Absorbed by Tube') #
    emissivity_tube = Float(0.5, iotype="in", units = 'W', desc='Emmissivity of the Tube') #
    sb_constant = Float(0.00000005670373, iotype="in", units = 'W/((m**2)*(K**4))', desc='Stefan-Boltzmann Constant') #

    #--Outputs--
    area_rad = Float(337486.1, units = 'm**2', iotype='out', desc='Tube Radiating Area') #
    #Required for Natural Convection Calcs
    GrDelTL3 = Float(1946216.7, units = '1/((ft**3)*F)', iotype='out', desc='Heat Radiated to the outside') #
    Pr = Float(0.707, iotype='out', desc='Heat Radiated to the outside') #
    Gr = Float(12730351223., iotype='out', desc='Heat Radiated to the outside') #
    Ra = Float(8996312085., iotype='out', desc='Heat Radiated to the outside') #
    Nu = Float(232.4543713, iotype='out', desc='Heat Radiated to the outside') #
    k = Float(0.02655, units = 'W/(m*K)', iotype='out', desc='Heat Radiated to the outside') #
    h = Float(0.845464094, units = 'W/((m**2)*K)', iotype='out', desc='Heat Radiated to the outside') #
    area_convection = Float(3374876.115, units = 'W', iotype='out', desc='Convection Area') #
    #Natural Convection
    q_per_area_nat_conv = Float(7.9, units = 'W/(m**2)', iotype='out', desc='Heat Radiated per Area to the outside') #
    total_q_nat_conv = Float(286900419., units = 'W', iotype='out', desc='Total Heat Radiated to the outside via Natural Convection') #
    #Exhausted from Pods
    heat_rate_pod = Float(519763, units = 'W', iotype='out', desc='Heating Due to a Single Pods') #
    total_heat_rate_pods = Float(17671942., units = 'W', iotype='out', desc='Heating Due to a All Pods') #
    #Radiated Out
    q_rad_per_area = Float(31.6, units = 'W/(m**2)', iotype='out', desc='Heat Radiated to the outside') #
    q_rad_tot = Float(106761066.5, units = 'W', iotype='out', desc='Heat Radiated to the outside') #
    #Radiated In
    viewing_angle = Float(1074256, units = 'm**2', iotype='out', desc='Effective Area hit by Sun') #
    #Total Heating
    q_total_out = Float(286900419., units = 'W', iotype='out', desc='Total Heat Released via Radiation and Natural Convection') #
    q_total_in = Float(286900419., units = 'W', iotype='out', desc='Total Heat Absorbed/Added via Pods and Solar Absorption') #
    #Residual (for solver)
    ss_temp_residual = Float(units = 'K', iotype='out', desc='Residual of T_released - T_absorbed')

    def execute(self):
        """Calculate Various Paramters"""

        bearing_q = cu(self.bearing_air.W,'lbm/s','kg/s') * cu(self.bearing_air.Cp,'Btu/(lbm*degR)','J/(kg*K)') * (cu(self.bearing_air.Tt,'degR','degK') - self.temp_boundary)
        nozzle_q = cu(self.nozzle_air.W,'lbm/s','kg/s') * cu(self.nozzle_air.Cp,'Btu/(lbm*degR)','J/(kg*K)') * (cu(self.nozzle_air.Tt,'degR','degK') - self.temp_boundary)
        #Q = mdot * cp * deltaT
        self.heat_rate_pod = nozzle_q +bearing_q
        #Total Q = Q * (number of pods)
        self.total_heat_rate_pods = self.heat_rate_pod*self.num_pods

        #Determine thermal resistance of outside via Natural Convection or forced convection
        if(self.temp_outside_ambient < 400):
            self.GrDelTL3 = 41780000000000000000*((self.temp_outside_ambient)**(-4.639)) #SI units (https://mdao.grc.nasa.gov/publications/Berton-Thesis.pdf pg51)
        else:
            self.GrDelTL3 = 4985000000000000000*((self.temp_outside_ambient)**(-4.284)) #SI units (https://mdao.grc.nasa.gov/publications/Berton-Thesis.pdf pg51)

        #Prandtl Number
        #Pr = viscous diffusion rate/ thermal diffusion rate = Cp * dyanamic viscosity / thermal conductivity
        #Pr << 1 means thermal diffusivity dominates
        #Pr >> 1 means momentum diffusivity dominates
        if (self.temp_outside_ambient < 400):
            self.Pr = 1.23*(self.temp_outside_ambient**(-0.09685)) #SI units (https://mdao.grc.nasa.gov/publications/Berton-Thesis.pdf pg51)
        else:
            self.Pr = 0.59*(self.temp_outside_ambient**(0.0239))
        #Grashof Number
        #Relationship between buoyancy and viscosity
        #Laminar = Gr < 10^8
        #Turbulent = Gr > 10^9
        self.Gr = self.GrDelTL3*(self.temp_boundary-self.temp_outside_ambient)*(self.diameter_outer_tube**3)
        #Rayleigh Number
        #Buoyancy driven flow (natural convection)
        self.Ra = self.Pr * self.Gr
        #Nusselt Number
        #Nu = convecive heat transfer / conductive heat transfer
        if (self.Ra<=10**12): #valid in specific flow regime
            self.Nu = (0.6 + 0.387*self.Ra**(1./6.)/(1 + (0.559/self.Pr)**(9./16.))**(8./27.))**2 #3rd Ed. of Introduction to Heat Transfer by Incropera and DeWitt, equations (9.33) and (9.34) on page 465
        if(self.temp_outside_ambient < 400):
            self.k = 0.0001423*(self.temp_outside_ambient**(0.9138)) #SI units (https://mdao.grc.nasa.gov/publications/Berton-Thesis.pdf pg51)
        else:
            self.k = 0.0002494*(self.temp_outside_ambient**(0.8152))
        #h = k*Nu/Characteristic Length
        self.h = (self.k * self.Nu)/ self.diameter_outer_tube
        #Convection Area = Surface Area
        self.area_convection = pi * self.length_tube * self.diameter_outer_tube
        #Determine heat radiated per square meter (Q)
        self.q_per_area_nat_conv = self.h*(self.temp_boundary-self.temp_outside_ambient)
        #Determine total heat radiated over entire tube (Qtotal)
        self.total_q_nat_conv = self.q_per_area_nat_conv * self.area_convection
        #Determine heat incoming via Sun radiation (Incidence Flux)
        #Sun hits an effective rectangular cross section
        self.area_viewing = self.length_tube* self.diameter_outer_tube
        self.q_per_area_solar = (1-self.surface_reflectance)* self.nn_incidence_factor * self.solar_insolation
        self.q_total_solar = self.q_per_area_solar * self.area_viewing
        #Determine heat released via radiation
        #Radiative area = surface area
        self.area_rad = self.area_convection
        #P/A = SB*emmisitivity*(T^4 - To^4)
        self.q_rad_per_area = self.sb_constant*self.emissivity_tube*((self.temp_boundary**4) - (self.temp_outside_ambient**4))
        #P = A * (P/A)
        self.q_rad_tot = self.area_rad * self.q_rad_per_area
        #------------
        #Sum Up
        self.q_total_out = self.q_rad_tot + self.total_q_nat_conv
        self.q_total_in = self.q_total_solar + self.total_heat_rate_pods

        self.ss_temp_residual = (self.q_total_out - self.q_total_in)/1e6

#run stand-alone component
if __name__ == "__main__":

    from openmdao.main.api import set_as_top

    class TubeHeatBalance(Assembly):

        def configure(self):

            tm = self.add('tm', TubeWall())
            #tm.bearing_air.setTotalTP()
            driver = self.add('driver',BroydenSolver())
            driver.add_parameter('tm.temp_boundary',low=0.,high=10000.)
            driver.add_constraint('tm.ss_temp_residual=0')

            driver.workflow.add(['tm'])

    test = TubeHeatBalance()
    set_as_top(test)

    #set input values
    test.tm.nozzle_air.setTotalTP(1710, 0.304434211)
    test.tm.nozzle_air.W = 1.08
    test.tm.bearing_air.W = 0.
    test.tm.diameter_outer_tube = 2.22504#, units = 'm', iotype='in', desc='Tube out diameter') #7.3ft
    test.tm.length_tube = 482803.#, units = 'm', iotype='in', desc='Length of entire Hyperloop') #300 miles, 1584000ft
    test.tm.num_pods = 34.#, units = 'K', iotype='in', desc='Number of Pods in the Tube at a given time') #
    test.tm.temp_boundary = 340#, units = 'K', iotype='in', desc='Average Temperature of the tube') #
    test.tm.temp_outside_ambient = 305.6#, units = 'K', iotype='in', desc='Average Temperature of the outside air') #

    test.run()

    print "-----Completed Tube Heat Flux Model Calculations---"
    print ""
    print "CompressQ-{} SolarQ-{} RadQ-{} ConvecQ-{}".format(test.tm.total_heat_rate_pods, test.tm.q_total_solar, test.tm.q_rad_tot, test.tm.total_q_nat_conv )
    print "Equilibrium Wall Temperature: {} K or {} F".format(test.tm.temp_boundary, cu(test.tm.temp_boundary,'degK','degF'))
    print "Ambient Temperature:          {} K or {} F".format(test.tm.temp_outside_ambient, cu(test.tm.temp_outside_ambient,'degK','degF'))
    print "Q Out = {} W  ==>  Q In = {} W ==> Error: {}%".format(test.tm.q_total_out,test.tm.q_total_in,((test.tm.q_total_out-test.tm.q_total_in)/test.tm.q_total_out)*100)<commit_msg>fixed wall temp name change<commit_after>"""
    tubeModel.py -
        Determines the steady state temperature of the hyperloop tube.
        Calculates Q released/absorbed by hyperloop tube due to:
        Internal Convection, Tube Conduction, Ambient Natural Convection, Solar Flux In, Radiation Out

    -original calculations from Jeff Berton, ported and extended by Jeff Chin

    Compatible with OpenMDAO v0.8.1
"""
from math import log, pi, sqrt, e

from openmdao.main.api import Assembly, Component
from openmdao.lib.drivers.api import BroydenSolver
from openmdao.lib.datatypes.api import Float, Bool
from openmdao.main.api import convert_units as cu

from pycycle.api import FlowStationVar

class TubeWallTemp(Component):
    """ Calculates Q released/absorbed by the hyperloop tube """
    #--Inputs--
    #Hyperloop Parameters/Design Variables
    diameter_outer_tube = Float(2.23, units = 'm', iotype='in', desc='tube outer diameter') #7.3ft
    length_tube = Float(482803, units = 'm', iotype='in', desc='Length of entire Hyperloop') #300 miles, 1584000ft
    num_pods = Float(34, units = 'K', iotype='in', desc='Number of Pods in the Tube at a given time') #
    temp_boundary = Float(322.0, units = 'K', iotype='in', desc='Average Temperature of the tube wall') #
    temp_outside_ambient = Float(305.6, units = 'K', iotype='in', desc='Average Temperature of the outside air') #
    nozzle_air = FlowStationVar(iotype="in", desc="air exiting the pod nozzle", copy=None)
    bearing_air = FlowStationVar(iotype="in", desc="air exiting the air bearings", copy=None)
    #constants
    solar_insolation = Float(1000., iotype="in", units = 'W/m**2', desc='solar irradiation at sea level on a clear day') #
    nn_incidence_factor = Float(0.7, iotype="in", desc='Non-normal incidence factor') #
    surface_reflectance = Float(0.5, desc='Solar Reflectance Index') #
    q_per_area_solar = Float(350., units = 'W/m**2', desc='Solar Heat Rate Absorbed per Area') #
    q_total_solar = Float(375989751., iotype="in", units = 'W', desc='Solar Heat Absorbed by Tube') #
    emissivity_tube = Float(0.5, iotype="in", units = 'W', desc='Emmissivity of the Tube') #
    sb_constant = Float(0.00000005670373, iotype="in", units = 'W/((m**2)*(K**4))', desc='Stefan-Boltzmann Constant') #

    #--Outputs--
    area_rad = Float(337486.1, units = 'm**2', iotype='out', desc='Tube Radiating Area') #
    #Required for Natural Convection Calcs
    GrDelTL3 = Float(1946216.7, units = '1/((ft**3)*F)', iotype='out', desc='Heat Radiated to the outside') #
    Pr = Float(0.707, iotype='out', desc='Heat Radiated to the outside') #
    Gr = Float(12730351223., iotype='out', desc='Heat Radiated to the outside') #
    Ra = Float(8996312085., iotype='out', desc='Heat Radiated to the outside') #
    Nu = Float(232.4543713, iotype='out', desc='Heat Radiated to the outside') #
    k = Float(0.02655, units = 'W/(m*K)', iotype='out', desc='Heat Radiated to the outside') #
    h = Float(0.845464094, units = 'W/((m**2)*K)', iotype='out', desc='Heat Radiated to the outside') #
    area_convection = Float(3374876.115, units = 'W', iotype='out', desc='Convection Area') #
    #Natural Convection
    q_per_area_nat_conv = Float(7.9, units = 'W/(m**2)', iotype='out', desc='Heat Radiated per Area to the outside') #
    total_q_nat_conv = Float(286900419., units = 'W', iotype='out', desc='Total Heat Radiated to the outside via Natural Convection') #
    #Exhausted from Pods
    heat_rate_pod = Float(519763, units = 'W', iotype='out', desc='Heating Due to a Single Pods') #
    total_heat_rate_pods = Float(17671942., units = 'W', iotype='out', desc='Heating Due to a All Pods') #
    #Radiated Out
    q_rad_per_area = Float(31.6, units = 'W/(m**2)', iotype='out', desc='Heat Radiated to the outside') #
    q_rad_tot = Float(106761066.5, units = 'W', iotype='out', desc='Heat Radiated to the outside') #
    #Radiated In
    viewing_angle = Float(1074256, units = 'm**2', iotype='out', desc='Effective Area hit by Sun') #
    #Total Heating
    q_total_out = Float(286900419., units = 'W', iotype='out', desc='Total Heat Released via Radiation and Natural Convection') #
    q_total_in = Float(286900419., units = 'W', iotype='out', desc='Total Heat Absorbed/Added via Pods and Solar Absorption') #
    #Residual (for solver)
    ss_temp_residual = Float(units = 'K', iotype='out', desc='Residual of T_released - T_absorbed')

    def execute(self):
        """Calculate Various Paramters"""

        bearing_q = cu(self.bearing_air.W,'lbm/s','kg/s') * cu(self.bearing_air.Cp,'Btu/(lbm*degR)','J/(kg*K)') * (cu(self.bearing_air.Tt,'degR','degK') - self.temp_boundary)
        nozzle_q = cu(self.nozzle_air.W,'lbm/s','kg/s') * cu(self.nozzle_air.Cp,'Btu/(lbm*degR)','J/(kg*K)') * (cu(self.nozzle_air.Tt,'degR','degK') - self.temp_boundary)
        #Q = mdot * cp * deltaT
        self.heat_rate_pod = nozzle_q +bearing_q
        #Total Q = Q * (number of pods)
        self.total_heat_rate_pods = self.heat_rate_pod*self.num_pods

        #Determine thermal resistance of outside via Natural Convection or forced convection
        if(self.temp_outside_ambient < 400):
            self.GrDelTL3 = 41780000000000000000*((self.temp_outside_ambient)**(-4.639)) #SI units (https://mdao.grc.nasa.gov/publications/Berton-Thesis.pdf pg51)
        else:
            self.GrDelTL3 = 4985000000000000000*((self.temp_outside_ambient)**(-4.284)) #SI units (https://mdao.grc.nasa.gov/publications/Berton-Thesis.pdf pg51)

        #Prandtl Number
        #Pr = viscous diffusion rate/ thermal diffusion rate = Cp * dyanamic viscosity / thermal conductivity
        #Pr << 1 means thermal diffusivity dominates
        #Pr >> 1 means momentum diffusivity dominates
        if (self.temp_outside_ambient < 400):
            self.Pr = 1.23*(self.temp_outside_ambient**(-0.09685)) #SI units (https://mdao.grc.nasa.gov/publications/Berton-Thesis.pdf pg51)
        else:
            self.Pr = 0.59*(self.temp_outside_ambient**(0.0239))
        #Grashof Number
        #Relationship between buoyancy and viscosity
        #Laminar = Gr < 10^8
        #Turbulent = Gr > 10^9
        self.Gr = self.GrDelTL3*(self.temp_boundary-self.temp_outside_ambient)*(self.diameter_outer_tube**3)
        #Rayleigh Number
        #Buoyancy driven flow (natural convection)
        self.Ra = self.Pr * self.Gr
        #Nusselt Number
        #Nu = convecive heat transfer / conductive heat transfer
        if (self.Ra<=10**12): #valid in specific flow regime
            self.Nu = (0.6 + 0.387*self.Ra**(1./6.)/(1 + (0.559/self.Pr)**(9./16.))**(8./27.))**2 #3rd Ed. of Introduction to Heat Transfer by Incropera and DeWitt, equations (9.33) and (9.34) on page 465
        if(self.temp_outside_ambient < 400):
            self.k = 0.0001423*(self.temp_outside_ambient**(0.9138)) #SI units (https://mdao.grc.nasa.gov/publications/Berton-Thesis.pdf pg51)
        else:
            self.k = 0.0002494*(self.temp_outside_ambient**(0.8152))
        #h = k*Nu/Characteristic Length
        self.h = (self.k * self.Nu)/ self.diameter_outer_tube
        #Convection Area = Surface Area
        self.area_convection = pi * self.length_tube * self.diameter_outer_tube
        #Determine heat radiated per square meter (Q)
        self.q_per_area_nat_conv = self.h*(self.temp_boundary-self.temp_outside_ambient)
        #Determine total heat radiated over entire tube (Qtotal)
        self.total_q_nat_conv = self.q_per_area_nat_conv * self.area_convection
        #Determine heat incoming via Sun radiation (Incidence Flux)
        #Sun hits an effective rectangular cross section
        self.area_viewing = self.length_tube* self.diameter_outer_tube
        self.q_per_area_solar = (1-self.surface_reflectance)* self.nn_incidence_factor * self.solar_insolation
        self.q_total_solar = self.q_per_area_solar * self.area_viewing
        #Determine heat released via radiation
        #Radiative area = surface area
        self.area_rad = self.area_convection
        #P/A = SB*emmisitivity*(T^4 - To^4)
        self.q_rad_per_area = self.sb_constant*self.emissivity_tube*((self.temp_boundary**4) - (self.temp_outside_ambient**4))
        #P = A * (P/A)
        self.q_rad_tot = self.area_rad * self.q_rad_per_area
        #------------
        #Sum Up
        self.q_total_out = self.q_rad_tot + self.total_q_nat_conv
        self.q_total_in = self.q_total_solar + self.total_heat_rate_pods

        self.ss_temp_residual = (self.q_total_out - self.q_total_in)/1e6

#run stand-alone component
if __name__ == "__main__":

    from openmdao.main.api import set_as_top

    class TubeHeatBalance(Assembly):

        def configure(self):

            tm = self.add('tm', TubeWallTemp())
            #tm.bearing_air.setTotalTP()
            driver = self.add('driver',BroydenSolver())
            driver.add_parameter('tm.temp_boundary',low=0.,high=10000.)
            driver.add_constraint('tm.ss_temp_residual=0')

            driver.workflow.add(['tm'])

    test = TubeHeatBalance()
    set_as_top(test)

    #set input values
    test.tm.nozzle_air.setTotalTP(1710, 0.304434211)
    test.tm.nozzle_air.W = 1.08
    test.tm.bearing_air.W = 0.
    test.tm.diameter_outer_tube = 2.22504#, units = 'm', iotype='in', desc='Tube out diameter') #7.3ft
    test.tm.length_tube = 482803.#, units = 'm', iotype='in', desc='Length of entire Hyperloop') #300 miles, 1584000ft
    test.tm.num_pods = 34.#, units = 'K', iotype='in', desc='Number of Pods in the Tube at a given time') #
    test.tm.temp_boundary = 340#, units = 'K', iotype='in', desc='Average Temperature of the tube') #
    test.tm.temp_outside_ambient = 305.6#, units = 'K', iotype='in', desc='Average Temperature of the outside air') #

    test.run()

    print "-----Completed Tube Heat Flux Model Calculations---"
    print ""
    print "CompressQ-{} SolarQ-{} RadQ-{} ConvecQ-{}".format(test.tm.total_heat_rate_pods, test.tm.q_total_solar, test.tm.q_rad_tot, test.tm.total_q_nat_conv )
    print "Equilibrium Wall Temperature: {} K or {} F".format(test.tm.temp_boundary, cu(test.tm.temp_boundary,'degK','degF'))
    print "Ambient Temperature:          {} K or {} F".format(test.tm.temp_outside_ambient, cu(test.tm.temp_outside_ambient,'degK','degF'))
    print "Q Out = {} W  ==>  Q In = {} W ==> Error: {}%".format(test.tm.q_total_out,test.tm.q_total_in,((test.tm.q_total_out-test.tm.q_total_in)/test.tm.q_total_out)*100)<|endoftext|>


<commit_before>"""Jinja2 templates for web pages."""

sed=('''<!DOCTYPE html>

        <meta charset="utf-8">
        <title>{{ main_id }}</title>
        {{ bokeh_css }}
        {{ bokeh_js }}

        {{ css }}

        {% if main_id %}

{{ main_id }}

                [

                {% if ra %}
                    α={{ ra|round(1) }}h, δ={{ dec|round(1) }}°
                {% endif %}

                {% if dist %}
                    | d={{ dist|round(1) }} pc
                {% endif %}

                ]

                home
                <form style="display:inline-block;" action = "/~grant/sdb/search/search.php" method = "GET">
                <input type = "text" name = "id"/>
                <input type = "submit" value = "Search" />
                </form>

            {% if sdbid %}
                data |
            {% endif %}

            simbad

            {% if ra %}
                | <a href="http://irsa.ipac.caltech.edu/applications/finderchart/#id=Hydra_finderchart_finder_chart&RequestClass=ServerRequest&DoSearch=true&subsize=0.083&thumbnail_size=medium&sources=DSS,SDSS,twomass,WISE,IRIS&overlay_catalog=true&catalog_by_radius=true&iras_radius=240&sdss_radius=5&twomass_radius=5&wise_radius=5&one_to_one=_none_&dss_bands=poss1_blue,poss1_red,poss2ukstu_blue,poss2ukstu_red,poss2ukstu_ir&SDSS_bands=u,g,r,i,z&twomass_bands=j,h,k&wise_bands=1,2,3,4&UserTargetWorldPt={{ ra*15 }};{{ dec }};EQ_J2000&projectId=finderchart&searchName=finder_chart&shortDesc=Finder%20Chart&isBookmarkAble=true&isDrillDownRoot=true&isSearchResult=true" target="_blank">finder
            {% endif %}

            {% if xids %}

                    {% for id in xids %}
                        {{ id }}
                    {% endfor %}

            {% endif %}

        {% endif %}

        {{ plot_div|indent(8) }}
        {{ plot_script|indent(8) }}

''')

generic=('''<!DOCTYPE html>

        <meta charset="utf-8">
        <title>{{ title }}</title>
        {{ bokeh_css }}
        {{ bokeh_js }}

        {{ css }}

        {% if title %}

{{ title }}

        {% endif %}

        {{ plot_div|indent(8) }}
        {{ plot_script|indent(8) }}

''')

generic_wide=('''<!DOCTYPE html>

        <meta charset="utf-8">
        <title>{{ title }}</title>
        {{ bokeh_css }}
        {{ bokeh_js }}

        {{ css }}

        {% if title %}

{{ title }}

        {% endif %}

        {{ plot_div|indent(8) }}
        {{ plot_script|indent(8) }}

''')

css=('''<link rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Lato">

        ''')
<commit_msg>spty and distance from parallax<commit_after>"""Jinja2 templates for web pages."""

index=('''<!DOCTYPE html>

        <meta charset="utf-8">
        <title>{{ main_id }}</title>
        {{ bokeh_css }}
        {{ bokeh_js }}

        {{ css }}

        {% if main_id %}

{{ main_id }}

                [

                {% if ra %}
                    α={{ (ra/15.0)|round(1) }}h, δ={{ dec|round(1) }}°
                {% endif %}

                {% if spty %}
                    | {{ spty }}
                {% endif %}

                {% if plx %}
                    | d={{ (1000.0/plx)|round(1) }} pc
                {% endif %}

                ]

                home
                <form style="display:inline-block;" action = "/~grant/sdb/search/search.php" method = "GET">
                <input type = "text" name = "id"/>
                <input type = "submit" value = "Search" />
                </form>

            {% if phot_file %}
                data |
            {% endif %}

            simbad

            {% if ra %}
                | <a href="http://irsa.ipac.caltech.edu/applications/finderchart/#id=Hydra_finderchart_finder_chart&RequestClass=ServerRequest&DoSearch=true&subsize=0.083&thumbnail_size=medium&sources=DSS,SDSS,twomass,WISE,IRIS&overlay_catalog=true&catalog_by_radius=true&iras_radius=240&sdss_radius=5&twomass_radius=5&wise_radius=5&one_to_one=_none_&dss_bands=poss1_blue,poss1_red,poss2ukstu_blue,poss2ukstu_red,poss2ukstu_ir&SDSS_bands=u,g,r,i,z&twomass_bands=j,h,k&wise_bands=1,2,3,4&UserTargetWorldPt={{ ra }};{{ dec }};EQ_J2000&projectId=finderchart&searchName=finder_chart&shortDesc=Finder%20Chart&isBookmarkAble=true&isDrillDownRoot=true&isSearchResult=true" target="_blank">finder
            {% endif %}

            {% if xids %}

                    {% for id in xids %}
                        {{ id }}
                    {% endfor %}

            {% endif %}

        {% endif %}

        {{ plot_div|indent(8) }}
        {{ plot_script|indent(8) }}

''')

generic=('''<!DOCTYPE html>

        <meta charset="utf-8">
        <title>{{ title }}</title>
        {{ bokeh_css }}
        {{ bokeh_js }}

        {{ css }}

        {% if title %}

{{ title }}

        {% endif %}

        {{ plot_div|indent(8) }}
        {{ plot_script|indent(8) }}

''')

generic_wide=('''<!DOCTYPE html>

        <meta charset="utf-8">
        <title>{{ title }}</title>
        {{ bokeh_css }}
        {{ bokeh_js }}

        {{ css }}

        {% if title %}

{{ title }}

        {% endif %}

        {{ plot_div|indent(8) }}
        {{ plot_script|indent(8) }}

''')

css=('''<link rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Lato">

        ''')
<|endoftext|>


<commit_before>"""
Library for helping SDK implementations
Version 0.2
"""

try:
    # Try to use the new Python3 HTTP library if available
    import http.client as http
    from urllib.parse import urlencode
except:
    # Must be using Python2 so use the appropriate library
    import httplib as http
    from urllib import urlencode

import ssl
import sys
import json
import collections

from .util.urlvalidator import urlsplit
from .util.decorators import deprecated

class SdkResponse(object):

    def __init__(self, data):
        try:
            self.data = json.loads(data)
        except ValueError:
            self.data = data

    def get_data(self):
        return self.data

class SdkBase(object):
    """

    """

    API_HOST = "127.0.0.1"
    API_PORT = 80
    API_SCHEME = "http"
    API_PROXY = None
    API_PROXY_PORT = 8080

    USER_AGENT_HEADER_NAME = "user-agent"
    PRAGMA_HEADER_NAME = "Pragma"
    CONTENT_TYPE_HEADER_NAME = "Content-Type"
    CONTENT_LENGTH_HEADER_NAME = "Content-Length"
    ACCEPT_HEADER_NAME = "Accept"

    CONTENT_TYPE_JSON = "application/json"
    CONTENT_TYPE_OCTET = "application/octet-stream"
    CONTENT_TYPE_PLAIN_TEXT = "text/plain; charset=utf-8"

    @classmethod
    def set_host(cls, hostname):
        """
        Set hostname.
        @param $host The host to be connected with, e.g. (http://hostname) or (https://X.X.X.X:port)
        """
        scheme, host, port = urlsplit(hostname)
        if scheme == 'http':
            cls.API_SCHEME = scheme
            cls.API_PORT = 80
        elif scheme == 'https':
            cls.API_SCHEME = scheme
            cls.API_PORT = 443
        if port:
            cls.API_PORT = port
        cls.API_HOST = host

    @classmethod
    def set_proxy(cls, proxy):
        """
        Set up proxy.
        @param $proxy The proxy server to be connected with, e.g. (http://hostname) or (https://hostname)
        """
        _, host, port = urlsplit(proxy)
        if port:
            cls.API_PROXY_PORT = port
        cls.API_PROXY = host

    @staticmethod
    def encode_multipart_formdata(fields, files):
        """
        fields is a sequence of (name, value) elements for regular form fields.
        files is a sequence of (name, value) elements for data to be uploaded as files
        Return (content_type, body) ready for httplib.HTTP instance
        """
        BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
        CRLF = '\r\n'
        L = []
        for (key, value) in fields:
            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name=%s' % key)
            L.append('Content-Type: %s' % SdkBase.CONTENT_TYPE_JSON)
            L.append('')
            L.append(value)
        for (key, value) in files:
            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name=%s' % (key))
            L.append('Content-Type: %s' % SdkBase.CONTENT_TYPE_OCTET)
            L.append('')
            L.append(value)
        L.append('--' + BOUNDARY + '--')
        L.append('')
        body = CRLF.join(L)
        content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
        return content_type, body

    @staticmethod
    def default_headers():
        return dict()

    def _http(self, method, url, headers=None, form_params=None, query_params=None, ssl_verified=False, form_urlencoding=True):
        """
        internal method to do http requests.
        :param method:
        :param url:
        :param headers:
        :param form_params:
        :return:
        """
        context = None

        if headers is not None:
            xHeaders = headers
        else:
            xHeaders = self.default_headers()

        if self.API_SCHEME == 'https':
            if sys.version_info[0:3] >= (2,7,9) or sys.version_info[0:3] >= (3,4,3):
                if not ssl_verified:
                    context = ssl._create_unverified_context()
                if self.API_PROXY:
                    conn = http.HTTPSConnection(self.API_PROXY, self.API_PROXY_PORT, context=context)
                    conn.set_tunnel(self.API_HOST, self.API_PORT)
                else:
                    conn = http.HTTPSConnection(self.API_HOST, self.API_PORT, context=context)
            else:
                if self.API_PROXY:
                    conn = http.HTTPSConnection(self.API_PROXY, self.API_PROXY_PORT)
                    conn.set_tunnel(self.API_HOST, self.API_PORT)
                else:
                    conn = http.HTTPSConnection(self.API_HOST, self.API_PORT)
        else:
            if self.API_PROXY:
                conn = http.HTTPConnection(self.API_PROXY, self.API_PROXY_PORT)
                url = "http://" + self.API_HOST + ":" + str(self.API_PORT) + url
            else:
                conn = http.HTTPConnection(self.API_HOST, self.API_PORT)

        if (method == "POST" or method == "PUT") and form_urlencoding:
            xHeaders["Content-Type"] = "application/x-www-form-urlencoded"

        if query_params:
            url += "?%s" % (urlencode(query_params))

        if form_params and form_urlencoding:
            parameters = urlencode(form_params)
            conn.request(method, url, parameters, headers=xHeaders)
        elif form_params and isinstance(form_params, collections.Mapping):
            parameters = json.dumps(form_params)
            conn.request(method, url, parameters, headers=xHeaders)
        elif form_params:
            parameters = form_params
            conn.request(method, url, parameters, headers=xHeaders)
        else:
            conn.request(method, url, headers=xHeaders)

        response = conn.getresponse()

        status = response.status
        resHeaders = response.getheaders()
        resData = response.read()
        #resData = response.read().encode('utf-8')

        conn.close()

        return status, resData, dict(resHeaders)

@deprecated
def parse_params(vars):
    to_return = dict()
    for elem in vars:
        if vars[elem] is not None:
            to_return[elem] = vars[elem]
    return to_return

@deprecated
def parse_args(**kwargs):
    to_return = dict()
    for elem in kwargs:
        if kwargs[elem] is not None:
            to_return[elem] = kwargs[elem]
    return to_return

@deprecated
def safe_add_slash(item):
    if item is not None:
        to_return = "/" + str(item)
    else:
        to_return = ""
    return to_return<commit_msg>add some header names<commit_after>"""
Library for helping SDK implementations
Version 0.2
"""

try:
    # Try to use the new Python3 HTTP library if available
    import http.client as http
    from urllib.parse import urlencode
except:
    # Must be using Python2 so use the appropriate library
    import httplib as http
    from urllib import urlencode

import ssl
import sys
import json
import collections

from .util.urlvalidator import urlsplit
from .util.decorators import deprecated

class SdkResponse(object):

    def __init__(self, data):
        try:
            self.data = json.loads(data)
        except ValueError:
            self.data = data

    def get_data(self):
        return self.data

class SdkBase(object):
    """

    """

    API_HOST = "127.0.0.1"
    API_PORT = 80
    API_SCHEME = "http"
    API_PROXY = None
    API_PROXY_PORT = 8080

    USER_AGENT_HEADER_NAME = "User-Agent"
    PRAGMA_HEADER_NAME = "Pragma"
    CONTENT_TYPE_HEADER_NAME = "Content-Type"
    CONTENT_LENGTH_HEADER_NAME = "Content-Length"
    ACCEPT_HEADER_NAME = "Accept"
    ACCEPT_LANGUAGE_HEADER_NAME = "Accept-Language"
    ACCEPT_ENCODING_HEADER_NAME = "Accept-Encoding"
    CACHE_CONTROL_HEADER_NAME = "Cache-Control"
    CONNECTION_HEADER_NAME = "Connection"
    REFERRER_HEADER_NAME = "Referer"

    CONTENT_TYPE_JSON = "application/json"
    CONTENT_TYPE_OCTET = "application/octet-stream"
    CONTENT_TYPE_PLAIN_TEXT = "text/plain; charset=utf-8"

    @classmethod
    def set_host(cls, hostname):
        """
        Set hostname.
        @param $host The host to be connected with, e.g. (http://hostname) or (https://X.X.X.X:port)
        """
        scheme, host, port = urlsplit(hostname)
        if scheme == 'http':
            cls.API_SCHEME = scheme
            cls.API_PORT = 80
        elif scheme == 'https':
            cls.API_SCHEME = scheme
            cls.API_PORT = 443
        if port:
            cls.API_PORT = port
        cls.API_HOST = host

    @classmethod
    def set_proxy(cls, proxy):
        """
        Set up proxy.
        @param $proxy The proxy server to be connected with, e.g. (http://hostname) or (https://hostname)
        """
        _, host, port = urlsplit(proxy)
        if port:
            cls.API_PROXY_PORT = port
        cls.API_PROXY = host

    @staticmethod
    def encode_multipart_formdata(fields, files):
        """
        fields is a sequence of (name, value) elements for regular form fields.
        files is a sequence of (name, value) elements for data to be uploaded as files
        Return (content_type, body) ready for httplib.HTTP instance
        """
        BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
        CRLF = '\r\n'
        L = []
        for (key, value) in fields:
            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name=%s' % key)
            L.append('Content-Type: %s' % SdkBase.CONTENT_TYPE_JSON)
            L.append('')
            L.append(value)
        for (key, value) in files:
            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name=%s' % (key))
            L.append('Content-Type: %s' % SdkBase.CONTENT_TYPE_OCTET)
            L.append('')
            L.append(value)
        L.append('--' + BOUNDARY + '--')
        L.append('')
        body = CRLF.join(L)
        content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
        return content_type, body

    @staticmethod
    def default_headers():
        return dict()

    def _http(self, method, url, headers=None, form_params=None, query_params=None, ssl_verified=False, form_urlencoding=True):
        """
        internal method to do http requests.
        :param method:
        :param url:
        :param headers:
        :param form_params:
        :return:
        """
        context = None

        if headers is not None:
            xHeaders = headers
        else:
            xHeaders = self.default_headers()

        if self.API_SCHEME == 'https':
            if sys.version_info[0:3] >= (2,7,9) or sys.version_info[0:3] >= (3,4,3):
                if not ssl_verified:
                    context = ssl._create_unverified_context()
                if self.API_PROXY:
                    conn = http.HTTPSConnection(self.API_PROXY, self.API_PROXY_PORT, context=context)
                    conn.set_tunnel(self.API_HOST, self.API_PORT)
                else:
                    conn = http.HTTPSConnection(self.API_HOST, self.API_PORT, context=context)
            else:
                if self.API_PROXY:
                    conn = http.HTTPSConnection(self.API_PROXY, self.API_PROXY_PORT)
                    conn.set_tunnel(self.API_HOST, self.API_PORT)
                else:
                    conn = http.HTTPSConnection(self.API_HOST, self.API_PORT)
        else:
            if self.API_PROXY:
                conn = http.HTTPConnection(self.API_PROXY, self.API_PROXY_PORT)
                url = "http://" + self.API_HOST + ":" + str(self.API_PORT) + url
            else:
                conn = http.HTTPConnection(self.API_HOST, self.API_PORT)

        if (method == "POST" or method == "PUT") and form_urlencoding:
            xHeaders["Content-Type"] = "application/x-www-form-urlencoded"

        if query_params:
            url += "?%s" % (urlencode(query_params))

        if form_params and form_urlencoding:
            parameters = urlencode(form_params)
            conn.request(method, url, parameters, headers=xHeaders)
        elif form_params and isinstance(form_params, collections.Mapping):
            parameters = json.dumps(form_params)
            conn.request(method, url, parameters, headers=xHeaders)
        elif form_params:
            parameters = form_params
            conn.request(method, url, parameters, headers=xHeaders)
        else:
            conn.request(method, url, headers=xHeaders)

        response = conn.getresponse()

        status = response.status
        resHeaders = response.getheaders()
        resData = response.read()
        #resData = response.read().encode('utf-8')

        conn.close()

        return status, resData, dict(resHeaders)

@deprecated
def parse_params(vars):
    to_return = dict()
    for elem in vars:
        if vars[elem] is not None:
            to_return[elem] = vars[elem]
    return to_return

@deprecated
def parse_args(**kwargs):
    to_return = dict()
    for elem in kwargs:
        if kwargs[elem] is not None:
            to_return[elem] = kwargs[elem]
    return to_return

@deprecated
def safe_add_slash(item):
    if item is not None:
        to_return = "/" + str(item)
    else:
        to_return = ""
    return to_return<|endoftext|>