<commit_before>"""
Original:
Beating the Benchmark
Search Results Relevance @ Kaggle
__author__ : Abhishek

Forked from Clubbing2Benchmark
__author__: Justfor

changed by
__author__: Henning Sperr

Modified by justfor
"""

import pandas as pd
import numpy as np
#from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.preprocessing import StandardScaler
from sklearn import  pipeline#, metrics, grid_search,decomposition,
from nltk.stem.porter import PorterStemmer
import re
from bs4 import BeautifulSoup
from sklearn.svm import SVC
from sklearn.feature_extraction import text
import string
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.metrics import accuracy_score,classification_report#,confusion_matrix

# array declarations
sw=[]
s_data = []
s_labels = []
t_data = []
t_labels = []
stemmer = PorterStemmer()
#stopwords tweak - more overhead
stop_words = ['http','www','img','border','color','style','padding','table','font','thi','inch','ha','width','height',
'0','1','2','3','4','5','6','7','8','9']
#stop_words = text.ENGLISH_STOP_WORDS.union(stop_words)

#stop_words = ['http','www','img','border','0','1','2','3','4','5','6','7','8','9']
stop_words = text.ENGLISH_STOP_WORDS.union(stop_words)

punct = string.punctuation
punct_re = re.compile('[{}]'.format(re.escape(punct)))

#remove html, remove non text or numeric, make query and title unique features for counts using prefix (accounted for in stopwords tweak)
stemmer = PorterStemmer()
## Stemming functionality
class stemmerUtility(object):
    """Stemming functionality"""
    @staticmethod
    def stemPorter(review_text):
        porter = PorterStemmer()
        preprocessed_docs = []
        for doc in review_text:
            final_doc = []
            for word in doc:
                final_doc.append(porter.stem(word))
                #final_doc.append(wordnet.lemmatize(word)) #note that lemmatize() can also takes part of speech as an argument!
            preprocessed_docs.append(final_doc)
        return preprocessed_docs

# The following 3 functions have been taken from Ben Hamner's github repository
# https://github.com/benhamner/Metrics
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
    """
    Returns the confusion matrix between rater's ratings
    """
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
        min_rating = min(rater_a + rater_b)
    if max_rating is None:
        max_rating = max(rater_a + rater_b)
    num_ratings = int(max_rating - min_rating + 1)
    conf_mat = [[0 for i in range(num_ratings)]
                for j in range(num_ratings)]
    for a, b in zip(rater_a, rater_b):
        conf_mat[a - min_rating][b - min_rating] += 1
    return conf_mat

def histogram(ratings, min_rating=None, max_rating=None):
    """
    Returns the counts of each type of rating that a rater made
    """
    if min_rating is None:
        min_rating = min(ratings)
    if max_rating is None:
        max_rating = max(ratings)
    num_ratings = int(max_rating - min_rating + 1)
    hist_ratings = [0 for x in range(num_ratings)]
    for r in ratings:
        hist_ratings[r - min_rating] += 1
    return hist_ratings

def quadratic_weighted_kappa(y, y_pred):
    """
    Calculates the quadratic weighted kappa
    axquadratic_weighted_kappa calculates the quadratic weighted kappa
    value, which is a measure of inter-rater agreement between two raters
    that provide discrete numeric ratings.  Potential values range from -1
    (representing complete disagreement) to 1 (representing complete
    agreement).  A kappa value of 0 is expected if all agreement is due to
    chance.
    quadratic_weighted_kappa(rater_a, rater_b), where rater_a and rater_b
    each correspond to a list of integer ratings.  These lists must have the
    same length.
    The ratings should be integers, and it is assumed that they contain
    the complete range of possible ratings.
    quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating
    is the minimum possible rating, and max_rating is the maximum possible
    rating
    """
    rater_a = y
    rater_b = y_pred
    min_rating=None
    max_rating=None
    rater_a = np.array(rater_a, dtype=int)
    rater_b = np.array(rater_b, dtype=int)
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
        min_rating = min(min(rater_a), min(rater_b))
    if max_rating is None:
        max_rating = max(max(rater_a), max(rater_b))
    conf_mat = confusion_matrix(rater_a, rater_b,
                                min_rating, max_rating)
    num_ratings = len(conf_mat)
    num_scored_items = float(len(rater_a))

    hist_rater_a = histogram(rater_a, min_rating, max_rating)
    hist_rater_b = histogram(rater_b, min_rating, max_rating)

    numerator = 0.0
    denominator = 0.0

    for i in range(num_ratings):
        for j in range(num_ratings):
            expected_count = (hist_rater_a[i] * hist_rater_b[j]
                              / num_scored_items)
            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)
            numerator += d * conf_mat[i][j] / num_scored_items
            denominator += d * expected_count / num_scored_items

    return (1.0 - numerator / denominator)

if __name__ == '__main__':

    # Load the training file
    train = pd.read_csv("input/train.csv").fillna("")
    test  = pd.read_csv("input/test.csv").fillna("")

    # we dont need ID columns
    idx = test.id.values.astype(int)

    # create labels. drop useless columns
    y = train.median_relevance.values

    def preprocess(x):
        trq=[]
        s=(" ").join(BeautifulSoup(x).get_text(" ").split(" "))
        s=re.sub("[^a-zA-Z]"," ", s)
        s= (" ").join([stemmer.stem(z) for z in s.split(" ") if len(z)>1])
        s=re.sub("[^a-zA-Z]+"," ", s)
        s=" ".join([t for t in s.lower().split(" ") if t not in stop_words and len(t)>1])
        for token in s.split(' '):
            trq.append(token)
        return ' '.join(trq)
    # Fit TFIDF
    import scipy.sparse
    def vectorize(train, tfv_query=None):
        query_data = list(train['query'].apply(preprocess))
        title_data = list(train['product_title'].apply(preprocess))
        if tfv_query is None:
            tfv_query = TfidfVectorizer(min_df=3,  max_features=None,
                    strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
                    ngram_range=(1, 3), use_idf=1,smooth_idf=1,sublinear_tf=1,
                    stop_words =None)

            full_data = query_data + title_data
            tfv_query.fit(full_data)

        return scipy.sparse.hstack([tfv_query.transform(query_data), tfv_query.transform(title_data)]), tfv_query

    X, tfv_query = vectorize(train)
    X_test, _ = vectorize(test, tfv_query)

    # Initialize SVD
    svd = TruncatedSVD(n_components=400)
    from sklearn.metrics.pairwise import linear_kernel
    class FeatureInserter():

        def __init__(self):
            pass

        def transform(self, X, y=None):
            distances = []
            quasi_jaccard = []
            print(len(distances), X.shape)

            for row in X.tocsr():
                row=row.toarray().ravel()
                cos_distance = linear_kernel(row[:row.shape[0]/2], row[row.shape[0]/2:])
                distances.append(cos_distance[0])
                intersect = row[:row.shape[0]/2].dot(row[row.shape[0]/2:])
                union = (row[:row.shape[0]/2]+row[row.shape[0]/2:]).dot((row[:row.shape[0]/2]+row[row.shape[0]/2:]))
                quasi_jaccard.append(1.0*intersect/union)

            print(len(distances), X.shape)
            print(distances[:10])

            #X = scipy.sparse.hstack([X, distances])
            return np.matrix([x for x in zip(distances, quasi_jaccard)])

        def fit(self, X,y):
            return self

        def fit_transform(self, X, y, **fit_params):
            self.fit(X,y)
            return self.transform(X)

    # Initialize the standard scaler
    scl = StandardScaler()

    # We will use SVM here..
    svm_model = SVC(C=10.)

    # Create the pipeline
    model = pipeline.Pipeline([('UnionInput', FeatureUnion([('svd', svd), ('dense_features', FeatureInserter())])),
    						 ('scl', scl),
                    	     ('svm', svm_model)])
    # Fit Model
    model.fit(X, y)

    preds = model.predict(X_test)
###################################Model performance ############
    yobs=y
    yhat1=model.predict(X)

    print 'Printing model1 performance'
#    print confusion_matrix(yobs, yhat),"\n",
    print accuracy_score(yobs,yhat1),"\n",classification_report(yobs,yhat1)
    from sklearn import cross_validation
    scores1 = cross_validation.cross_val_score(model, X, y, cv=5,scoring='f1_weighted')
    print round(np.mean(scores1),4),"\n",scores1

############################################################################################

    for i in range(len(train.id)):
        s=(" ").join(["q"+ z for z in BeautifulSoup(train["query"][i]).get_text(" ").split(" ")]) + " " + (" ").join(["z"+ z for z in BeautifulSoup(train.product_title[i]).get_text(" ").split(" ")]) + " " + BeautifulSoup(train.product_description[i]).get_text(" ")
        s=re.sub("[^a-zA-Z0-9]"," ", s)
        s= (" ").join([stemmer.stem(z) for z in s.split(" ")])
        s_data.append(s)
        s_labels.append(str(train["median_relevance"][i]))
    for i in range(len(test.id)):
        s=(" ").join(["q"+ z for z in BeautifulSoup(test["query"][i]).get_text().split(" ")]) + " " + (" ").join(["z"+ z for z in BeautifulSoup(test.product_title[i]).get_text().split(" ")]) + " " + BeautifulSoup(test.product_description[i]).get_text()
        s=re.sub("[^a-zA-Z0-9]"," ", s)
        s= (" ").join([stemmer.stem(z) for z in s.split(" ")])
        t_data.append(s)
    #create sklearn pipeline, fit all, and predit test data
    clf = Pipeline([('v',TfidfVectorizer(min_df=5, max_df=500, max_features=None, strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}', ngram_range=(1, 2), use_idf=True, smooth_idf=True, sublinear_tf=True, stop_words = 'english')),
    ('svd', TruncatedSVD(n_components=300, algorithm='randomized', n_iter=5, random_state=None, tol=0.0)),
    ('scl', StandardScaler(copy=True, with_mean=True, with_std=True)),
    ('svm', SVC(C=10.0, kernel='rbf', degree=3, gamma=0.0, coef0=0.0, shrinking=True, probability=False, tol=0.001, cache_size=200, class_weight=None, verbose=False, max_iter=-1, random_state=None))])
    clf.fit(s_data, s_labels)
    t_labels = clf.predict(t_data)

###################################Model performance ############
    yobs=s_labels
    yhat2=clf.predict(s_data)

    print 'Printing model2 performance'
#    print confusion_matrix(yobs, yhat),"\n",
    print accuracy_score(yobs,yhat2),"\n",classification_report(yobs,yhat2)
#    from sklearn import cross_validation
    scores2 = cross_validation.cross_val_score(clf, s_data, s_labels, cv=5,scoring='f1_weighted')
    print round(np.mean(scores2),4),"\n",scores2

############################################################################################

    import math
    p3 = []
    for i in range(len(preds)):
        x =math.sqrt((int(t_labels[i]))*preds[i])
        x = round(x)
        p3.append(int(x))
    ##### Tell the performance of ensemble #######
    yhat2=yhat2.astype(int)
    yobs=np.asarray(yobs,dtype=int)
    yhat=(yhat1+yhat2)/2
    print 'Ensemble by AM',accuracy_score(yobs,yhat),"\n",classification_report(yobs,yhat)
    yhat=np.sqrt(yhat1*yhat2)
    yhat=yhat.astype(int)
#    yhat = round(yhat)
    print 'Ensemble by GM',accuracy_score(yobs,yhat),"\n",classification_report(yobs,yhat)

    # Create your first submission file
    submission = pd.DataFrame({"id": idx, "prediction": p3})
    submission.to_csv("ensemble_svc_different_preproc.csv", index=False)

<commit_msg>Optimized output<commit_after>"""
Original:
Beating the Benchmark
Search Results Relevance @ Kaggle
__author__ : Abhishek

Forked from Clubbing2Benchmark
__author__: Justfor

changed by
__author__: Henning Sperr

Modified by justfor
"""

import pandas as pd
import numpy as np
#from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.preprocessing import StandardScaler
from sklearn import  pipeline#, metrics, grid_search,decomposition,
from nltk.stem.porter import PorterStemmer
import re
from bs4 import BeautifulSoup
from sklearn.svm import SVC
from sklearn.feature_extraction import text
import string
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.metrics import accuracy_score,classification_report#,confusion_matrix
from sklearn.multiclass import OneVsRestClassifier,OneVsOneClassifier,OutputCodeClassifier

# array declarations
sw=[]
s_data = []
s_labels = []
t_data = []
t_labels = []
stemmer = PorterStemmer()
#stopwords tweak - more overhead
stop_words = ['http','www','img','border','color','style','padding','table','font','thi','inch','ha','width','height',
'0','1','2','3','4','5','6','7','8','9']
#stop_words = text.ENGLISH_STOP_WORDS.union(stop_words)

#stop_words = ['http','www','img','border','0','1','2','3','4','5','6','7','8','9']
stop_words = text.ENGLISH_STOP_WORDS.union(stop_words)

punct = string.punctuation
punct_re = re.compile('[{}]'.format(re.escape(punct)))

#remove html, remove non text or numeric, make query and title unique features for counts using prefix (accounted for in stopwords tweak)
stemmer = PorterStemmer()
## Stemming functionality
class stemmerUtility(object):
    """Stemming functionality"""
    @staticmethod
    def stemPorter(review_text):
        porter = PorterStemmer()
        preprocessed_docs = []
        for doc in review_text:
            final_doc = []
            for word in doc:
                final_doc.append(porter.stem(word))
                #final_doc.append(wordnet.lemmatize(word)) #note that lemmatize() can also takes part of speech as an argument!
            preprocessed_docs.append(final_doc)
        return preprocessed_docs

# The following 3 functions have been taken from Ben Hamner's github repository
# https://github.com/benhamner/Metrics
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
    """
    Returns the confusion matrix between rater's ratings
    """
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
        min_rating = min(rater_a + rater_b)
    if max_rating is None:
        max_rating = max(rater_a + rater_b)
    num_ratings = int(max_rating - min_rating + 1)
    conf_mat = [[0 for i in range(num_ratings)]
                for j in range(num_ratings)]
    for a, b in zip(rater_a, rater_b):
        conf_mat[a - min_rating][b - min_rating] += 1
    return conf_mat

def histogram(ratings, min_rating=None, max_rating=None):
    """
    Returns the counts of each type of rating that a rater made
    """
    if min_rating is None:
        min_rating = min(ratings)
    if max_rating is None:
        max_rating = max(ratings)
    num_ratings = int(max_rating - min_rating + 1)
    hist_ratings = [0 for x in range(num_ratings)]
    for r in ratings:
        hist_ratings[r - min_rating] += 1
    return hist_ratings

def quadratic_weighted_kappa(y, y_pred):
    """
    Calculates the quadratic weighted kappa
    axquadratic_weighted_kappa calculates the quadratic weighted kappa
    value, which is a measure of inter-rater agreement between two raters
    that provide discrete numeric ratings.  Potential values range from -1
    (representing complete disagreement) to 1 (representing complete
    agreement).  A kappa value of 0 is expected if all agreement is due to
    chance.
    quadratic_weighted_kappa(rater_a, rater_b), where rater_a and rater_b
    each correspond to a list of integer ratings.  These lists must have the
    same length.
    The ratings should be integers, and it is assumed that they contain
    the complete range of possible ratings.
    quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating
    is the minimum possible rating, and max_rating is the maximum possible
    rating
    """
    rater_a = y
    rater_b = y_pred
    min_rating=None
    max_rating=None
    rater_a = np.array(rater_a, dtype=int)
    rater_b = np.array(rater_b, dtype=int)
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
        min_rating = min(min(rater_a), min(rater_b))
    if max_rating is None:
        max_rating = max(max(rater_a), max(rater_b))
    conf_mat = confusion_matrix(rater_a, rater_b,
                                min_rating, max_rating)
    num_ratings = len(conf_mat)
    num_scored_items = float(len(rater_a))

    hist_rater_a = histogram(rater_a, min_rating, max_rating)
    hist_rater_b = histogram(rater_b, min_rating, max_rating)

    numerator = 0.0
    denominator = 0.0

    for i in range(num_ratings):
        for j in range(num_ratings):
            expected_count = (hist_rater_a[i] * hist_rater_b[j]
                              / num_scored_items)
            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)
            numerator += d * conf_mat[i][j] / num_scored_items
            denominator += d * expected_count / num_scored_items

    return (1.0 - numerator / denominator)

if __name__ == '__main__':

    # Load the training file
    train = pd.read_csv("input/train.csv").fillna("")
    test  = pd.read_csv("input/test.csv").fillna("")

    # we dont need ID columns
    idx = test.id.values.astype(int)

    # create labels. drop useless columns
    y = train.median_relevance.values

    def preprocess(x):
        trq=[]
        s=(" ").join(BeautifulSoup(x).get_text(" ").split(" "))
        s=re.sub("[^a-zA-Z]"," ", s)
        s= (" ").join([stemmer.stem(z) for z in s.split(" ") if len(z)>1])
        s=re.sub("[^a-zA-Z]+"," ", s)
        s=" ".join([t for t in s.lower().split(" ") if t not in stop_words and len(t)>1])
        for token in s.split(' '):
            trq.append(token)
        return ' '.join(trq)
    # Fit TFIDF
    import scipy.sparse
    def vectorize(train, tfv_query=None):
        query_data = list(train['query'].apply(preprocess))
        title_data = list(train['product_title'].apply(preprocess))
        if tfv_query is None:
            tfv_query = TfidfVectorizer(min_df=3,  max_features=None,
                    strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
                    ngram_range=(1, 3), use_idf=1,smooth_idf=1,sublinear_tf=1,
                    stop_words =None)

            full_data = query_data + title_data
            tfv_query.fit(full_data)

        return scipy.sparse.hstack([tfv_query.transform(query_data), tfv_query.transform(title_data)]), tfv_query

    X, tfv_query = vectorize(train)
    X_test, _ = vectorize(test, tfv_query)

    # Initialize SVD
    svd = TruncatedSVD(n_components=450,algorithm='randomized')
    from sklearn.metrics.pairwise import linear_kernel
    class FeatureInserter():

        def __init__(self):
            pass

        def transform(self, X, y=None):
            distances = []
            quasi_jaccard = []
            print(len(distances), X.shape)

            for row in X.tocsr():
                row=row.toarray().ravel()
                cos_distance = linear_kernel(row[:row.shape[0]/2], row[row.shape[0]/2:])
                distances.append(cos_distance[0])
                intersect = row[:row.shape[0]/2].dot(row[row.shape[0]/2:])
                union = (row[:row.shape[0]/2]+row[row.shape[0]/2:]).dot((row[:row.shape[0]/2]+row[row.shape[0]/2:]))
                quasi_jaccard.append(1.0*intersect/union)

            print(len(distances), X.shape)
            print(distances[:10])

            #X = scipy.sparse.hstack([X, distances])
            return np.matrix([x for x in zip(distances, quasi_jaccard)])

        def fit(self, X,y):
            return self

        def fit_transform(self, X, y, **fit_params):
            self.fit(X,y)
            return self.transform(X)

    # Initialize the standard scaler
    scl = StandardScaler()

    # We will use SVM here..
    svm_model = OutputCodeClassifier(SVC(C=10.),code_size=2, random_state=0)

    # Create the pipeline
    model = pipeline.Pipeline([('UnionInput', FeatureUnion([('svd', svd), ('dense_features', FeatureInserter())])),
    						 ('scl', scl),
                    	     ('svm', svm_model)])
    # Fit Model
    model.fit(X, y)

    preds = model.predict(X_test)
###################################Model performance ############
    yobs=y
    yhat1=model.predict(X)

    print 'Printing model1 performance'
#    print confusion_matrix(yobs, yhat),"\n",
    print accuracy_score(yobs,yhat1),"\n",classification_report(yobs,yhat1)
    from sklearn import cross_validation
    scores1 = cross_validation.cross_val_score(model, X, y, cv=5,scoring='f1_weighted')
    print round(np.mean(scores1),4),"\n",scores1

############################################################################################

    for i in range(len(train.id)):
        s=(" ").join(["q"+ z for z in BeautifulSoup(train["query"][i]).get_text(" ").split(" ")]) + " " + (" ").join(["z"+ z for z in BeautifulSoup(train.product_title[i]).get_text(" ").split(" ")]) + " " + BeautifulSoup(train.product_description[i]).get_text(" ")
        s=re.sub("[^a-zA-Z0-9]"," ", s)
        s= (" ").join([stemmer.stem(z) for z in s.split(" ")])
        s_data.append(s)
        s_labels.append(str(train["median_relevance"][i]))
    for i in range(len(test.id)):
        s=(" ").join(["q"+ z for z in BeautifulSoup(test["query"][i]).get_text().split(" ")]) + " " + (" ").join(["z"+ z for z in BeautifulSoup(test.product_title[i]).get_text().split(" ")]) + " " + BeautifulSoup(test.product_description[i]).get_text()
        s=re.sub("[^a-zA-Z0-9]"," ", s)
        s= (" ").join([stemmer.stem(z) for z in s.split(" ")])
        t_data.append(s)
    #create sklearn pipeline, fit all, and predit test data
    clf = Pipeline([('v',TfidfVectorizer(min_df=5, max_df=500, max_features=None, strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}', ngram_range=(1, 2), use_idf=True, smooth_idf=True, sublinear_tf=True, stop_words = 'english')),
    ('svd', TruncatedSVD(n_components=450, algorithm='randomized', n_iter=5, random_state=None, tol=0.0)),
    ('scl', StandardScaler(copy=True, with_mean=True, with_std=True)),
    ('svm', OutputCodeClassifier(SVC(C=10.0, kernel='rbf', degree=3, gamma=0.0, coef0=0.0, shrinking=True, probability=False, tol=0.001, cache_size=200, class_weight=None, verbose=False, max_iter=-1, random_state=None),code_size=2, random_state=0))])
    clf.fit(s_data, s_labels)
    t_labels = clf.predict(t_data)

###################################Model performance ############
    yobs=s_labels
    yhat2=clf.predict(s_data)

    print 'Printing model2 performance'
#    print confusion_matrix(yobs, yhat),"\n",
    print accuracy_score(yobs,yhat2),"\n",classification_report(yobs,yhat2)
#    from sklearn import cross_validation
    scores2 = cross_validation.cross_val_score(clf, s_data, s_labels, cv=5,scoring='f1_weighted')
    print round(np.mean(scores2),4),"\n",scores2

############################################################################################
#    Changed n_component from 200 to 300 for model 2 # Result got better by 2%
#    Changed n_component from 400 to 300 for model 1 # Result got worse by 2%
#    Changed n_component from 400 to 450 for model 1 # Result got better by 3.5%
#    Changed n_component from 450 to 500 for model 1 # Result got better by 0.3%
#    Changed n_component from 300 to 400 for model 2 # Result got better to 0.951
#    however cv score is down
#    Changed model2 n_component from 400 to 450# GM 0.956 Model cv scores: 0.651 0.6264
#    Changed model2 C=11 GM 0.956 Model cv scores: 0.6482 0.6218
#    Model2 max_df 500 to 600 GM 0.956 Model cv scores: 0.6504 0.6246
#    Model2 degree 3 to 4 GM 0.954 Model cv scores: 0.6504 0.6233
#    Model2 gamma 0 to 0.001 GM 0.915 Model cv scores: 0.6504 0.6294
#    Model2 algorithm randomized to algorithm='arpack': GM 0.9530  Model cv scores: 0.6484 0.6256
#    450 & 450 # Model1 0.6486 0.9191 model2 0.6241  0.9569 GM 0.9532
#    Model2 onevsrest 0.6248 0.96022
#    Model2 OneVsOne
#    Model2 OutputCode

############################################################################################
    import math
    p3 = []
    for i in range(len(preds)):
        x =math.sqrt((int(t_labels[i]))*preds[i])
        x = round(x)
        p3.append(int(x))
    ##### Tell the performance of ensemble #######
    yhat2=yhat2.astype(int)
    yobs=np.asarray(yobs,dtype=int)
    yhat=(yhat1+yhat2)/2
    print 'Ensemble by AM',accuracy_score(yobs,yhat),"\n",classification_report(yobs,yhat)
    yhat=np.sqrt(yhat1*yhat2)
    yhat=yhat.astype(int)
#    yhat = round(yhat)
    print 'Ensemble by GM',accuracy_score(yobs,yhat),"\n",classification_report(yobs,yhat)
    print 'Model cv scores:',round(np.mean(scores1),4),round(np.mean(scores2),4)
    # Create your first submission file
    submission = pd.DataFrame({"id": idx, "prediction": p3})
    submission.to_csv("ensemble_svc_450_450_oe.csv", index=False)

<|endoftext|>


<commit_before># Copyright 2012 NEC Corporation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Views for managing Neutron Networks.
"""
from django.conf import settings
from django.core.urlresolvers import reverse
from django.core.urlresolvers import reverse_lazy
from django.utils.translation import ugettext_lazy as _

from horizon import exceptions
from horizon import forms
from horizon import tables
from horizon import tabs
from horizon.utils import memoized
from horizon import workflows

from openstack_dashboard import api
from openstack_dashboard.utils import filters

from openstack_dashboard.dashboards.project.networks \
    import forms as project_forms
from openstack_dashboard.dashboards.project.networks \
    import tables as project_tables
from openstack_dashboard.dashboards.project.networks import tabs \
    as network_tabs
from openstack_dashboard.dashboards.project.networks \
    import workflows as project_workflows

class IndexView(tables.DataTableView):
    table_class = project_tables.NetworksTable
    page_title = _("Networks")
    FILTERS_MAPPING = {'shared': {_("yes"): True, _("no"): False},
                       'router:external': {_("yes"): True, _("no"): False},
                       'admin_state_up': {_("up"): True, _("down"): False}}

    def get_data(self):
        try:
            tenant_id = self.request.user.tenant_id
            search_opts = self.get_filters(filters_map=self.FILTERS_MAPPING)
            networks = api.neutron.network_list_for_tenant(
                self.request, tenant_id, include_external=True, **search_opts)
        except Exception:
            networks = []
            msg = _('Network list can not be retrieved.')
            exceptions.handle(self.request, msg)
        return networks

class DefaultSubnetWorkflowMixin(object):

    def get_default_dns_servers(self):
        # this returns the default dns servers to be used for new subnets
        dns_default = "\n".join(getattr(settings, 'OPENSTACK_NEUTRON_NETWORK',
                                        {}).get('default_dns_nameservers', ''))
        return dns_default

class CreateView(DefaultSubnetWorkflowMixin, workflows.WorkflowView):
    workflow_class = project_workflows.CreateNetwork

    def get_initial(self):
        results = super(CreateView, self).get_initial()
        results['dns_nameservers'] = self.get_default_dns_servers()
        return results

class UpdateView(forms.ModalFormView):
    context_object_name = 'network'
    form_class = project_forms.UpdateNetwork
    form_id = "update_network_form"
    submit_label = _("Save Changes")
    submit_url = "horizon:project:networks:update"
    success_url = reverse_lazy("horizon:project:networks:index")
    template_name = 'project/networks/update.html'
    page_title = _("Edit Network")

    def get_context_data(self, **kwargs):
        context = super(UpdateView, self).get_context_data(**kwargs)
        args = (self.kwargs['network_id'],)
        context["network_id"] = self.kwargs['network_id']
        context["submit_url"] = reverse(self.submit_url, args=args)
        return context

    @memoized.memoized_method
    def _get_object(self, *args, **kwargs):
        network_id = self.kwargs['network_id']
        try:
            # no subnet values are read or editable in this view, so
            # save the subnet expansion overhead
            return api.neutron.network_get(self.request, network_id,
                                           expand_subnet=False)
        except Exception:
            redirect = self.success_url
            msg = _('Unable to retrieve network details.')
            exceptions.handle(self.request, msg, redirect=redirect)

    def get_initial(self):
        network = self._get_object()
        return {'network_id': network['id'],
                'tenant_id': network['tenant_id'],
                'name': network['name'],
                'admin_state': network['admin_state_up'],
                'shared': network['shared']}

class DetailView(tabs.TabbedTableView):
    tab_group_class = network_tabs.NetworkDetailsTabs
    template_name = 'horizon/common/_detail.html'
    page_title = '{{ network.name | default:network.id }}'

    @staticmethod
    def get_redirect_url():
        return reverse('horizon:project:networks:index')

    @memoized.memoized_method
    def _get_data(self):
        try:
            network_id = self.kwargs['network_id']
            network = api.neutron.network_get(self.request, network_id)
            network.set_id_as_name_if_empty(length=0)
        except Exception:
            msg = _('Unable to retrieve details for network "%s".') \
                % (network_id)
            exceptions.handle(self.request, msg,
                              redirect=self.get_redirect_url())
        return network

    def get_subnets_data(self):
        # MultiTableMixin requires this method to be defined
        pass

    def get_context_data(self, **kwargs):
        context = super(DetailView, self).get_context_data(**kwargs)
        network = self._get_data()
        context["network"] = network
        table = project_tables.NetworksTable(self.request)
        context["url"] = self.get_redirect_url()
        context["actions"] = table.render_row_actions(network)
        choices = project_tables.STATUS_DISPLAY_CHOICES
        network.status_label = (
            filters.get_display_label(choices, network.status))
        choices = project_tables.DISPLAY_CHOICES
        network.admin_state_label = (
            filters.get_display_label(choices, network.admin_state))
        return context
<commit_msg>FIX: network views<commit_after># Copyright 2012 NEC Corporation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Views for managing Neutron Networks.
"""
from django.conf import settings
from django.core.urlresolvers import reverse
from django.core.urlresolvers import reverse_lazy
from django.utils.translation import ugettext_lazy as _

from horizon import exceptions
from horizon import forms
from horizon import tables
from horizon import tabs
from horizon.utils import memoized
from horizon import workflows

from openstack_dashboard import api
from openstack_dashboard.utils import filters

from openstack_dashboard.dashboards.project.networks \
    import forms as project_forms
from openstack_dashboard.dashboards.project.networks \
    import tables as project_tables
from openstack_dashboard.dashboards.project.networks import tabs \
    as network_tabs
from openstack_dashboard.dashboards.project.networks \
    import workflows as project_workflows

class IndexView(tables.DataTableView):
    table_class = project_tables.NetworksTable
    page_title = _("Networks")
    FILTERS_MAPPING = {'shared': {_("yes"): True, _("no"): False},
                       'router:external': {_("yes"): True, _("no"): False},
                       'admin_state_up': {_("up"): True, _("down"): False}}

    def get_data(self):
        try:
            tenant_id = self.request.user.tenant_id
            search_opts = self.get_filters(filters_map=self.FILTERS_MAPPING)
            networks = api.neutron.network_list_for_tenant(
                self.request, tenant_id, include_external=True, **search_opts)
        except Exception:
            networks = []
            msg = _('Network list can not be retrieved.')
            exceptions.handle(self.request, msg)

        fix_networks = []
        for network in networks:
            if (not ("PLSK" in network.name)) and (not ("PSK" in network.name)):
                fix_networks.append(network)

        #return networks
        return fix_networks

class DefaultSubnetWorkflowMixin(object):

    def get_default_dns_servers(self):
        # this returns the default dns servers to be used for new subnets
        dns_default = "\n".join(getattr(settings, 'OPENSTACK_NEUTRON_NETWORK',
                                        {}).get('default_dns_nameservers', ''))
        return dns_default

class CreateView(DefaultSubnetWorkflowMixin, workflows.WorkflowView):
    workflow_class = project_workflows.CreateNetwork

    def get_initial(self):
        results = super(CreateView, self).get_initial()
        results['dns_nameservers'] = self.get_default_dns_servers()
        return results

class UpdateView(forms.ModalFormView):
    context_object_name = 'network'
    form_class = project_forms.UpdateNetwork
    form_id = "update_network_form"
    submit_label = _("Save Changes")
    submit_url = "horizon:project:networks:update"
    success_url = reverse_lazy("horizon:project:networks:index")
    template_name = 'project/networks/update.html'
    page_title = _("Edit Network")

    def get_context_data(self, **kwargs):
        context = super(UpdateView, self).get_context_data(**kwargs)
        args = (self.kwargs['network_id'],)
        context["network_id"] = self.kwargs['network_id']
        context["submit_url"] = reverse(self.submit_url, args=args)
        return context

    @memoized.memoized_method
    def _get_object(self, *args, **kwargs):
        network_id = self.kwargs['network_id']
        try:
            # no subnet values are read or editable in this view, so
            # save the subnet expansion overhead
            return api.neutron.network_get(self.request, network_id,
                                           expand_subnet=False)
        except Exception:
            redirect = self.success_url
            msg = _('Unable to retrieve network details.')
            exceptions.handle(self.request, msg, redirect=redirect)

    def get_initial(self):
        network = self._get_object()
        return {'network_id': network['id'],
                'tenant_id': network['tenant_id'],
                'name': network['name'],
                'admin_state': network['admin_state_up'],
                'shared': network['shared']}

class DetailView(tabs.TabbedTableView):
    tab_group_class = network_tabs.NetworkDetailsTabs
    template_name = 'horizon/common/_detail.html'
    page_title = '{{ network.name | default:network.id }}'

    @staticmethod
    def get_redirect_url():
        return reverse('horizon:project:networks:index')

    @memoized.memoized_method
    def _get_data(self):
        try:
            network_id = self.kwargs['network_id']
            network = api.neutron.network_get(self.request, network_id)
            network.set_id_as_name_if_empty(length=0)
        except Exception:
            msg = _('Unable to retrieve details for network "%s".') \
                % (network_id)
            exceptions.handle(self.request, msg,
                              redirect=self.get_redirect_url())
        return network

    def get_subnets_data(self):
        # MultiTableMixin requires this method to be defined
        pass

    def get_context_data(self, **kwargs):
        context = super(DetailView, self).get_context_data(**kwargs)
        network = self._get_data()
        context["network"] = network
        table = project_tables.NetworksTable(self.request)
        context["url"] = self.get_redirect_url()
        context["actions"] = table.render_row_actions(network)
        choices = project_tables.STATUS_DISPLAY_CHOICES
        network.status_label = (
            filters.get_display_label(choices, network.status))
        choices = project_tables.DISPLAY_CHOICES
        network.admin_state_label = (
            filters.get_display_label(choices, network.admin_state))
        return context
<|endoftext|>


<commit_before>from django.db import models
from django.contrib.auth.models import User
from sublist.models import Sublist

'''
A player can exist on many teams and in many divisions both as a division rep and/or as a sub, and these
are not mutually exlusive
Player -> Team     : 0 .. *
Player -> Division : 0 .. *
Player -> Sub      :
'''
class Player(models.Model):
	user         = models.OneToOneField(
	    User,
            on_delete=models.CASCADE,
            primary_key=True,
	    )
	phone_number = models.IntegerField(blank=True, null=True)

	def __str__(self):
# chose not to use this because fName and lName are not required on users yet.
#		name = self.user.first_name + ' ' + self.user.last_name
		return self.user.username

	#function to return all of the sublists related to a player instance
	def related_sublists(self):
            return Sublist.objects.filter(session__subs__player=self)

'''
                sessions = None
		sublists = None
		subs = self.sub_set.all()
		for sub in subs:
			sessions += sub.sessions
		for session in sessions:
			sublists += session.sublist
		return sublists
		if self.sub:
			sub = self.sub
			related_sessions = sub.session_set.all()
			for session in related_sessoins:
				sublists += session.sublist
			return sublists
'''

'''
Subs are a "tuple-ish" construction tying a player and a date together to indicate what date they are
willing to sub in a Division.

Note: Eventually, I'd like players to be able to select a particular date that they could sub in a
Division, OR, put themselves down for an entire session, in which case, we will likely need a model for
session containing the dates of a particular session within a division. There are typically 3 sessions
in a division per year, but the sessions don't necessarily correspond across divisions.
'''
class Sub(models.Model):
	player = models.ForeignKey(Player)
	date   = models.DateTimeField('sub date')

	def __str__(self):
		availability = self.player.user.username + ' is available ' + str(self.date)
		return availability

	def _get_sessions(self):
		return self.session_set.all()
	sessions = property(_get_sessions)

'''
A team can contain many players but should only ever exist in one division
Team -> Player   : 1 .. *
Team -> Division : 1
'''
class Team(models.Model):
	name          = models.CharField(max_length=200)
#	division      = models.ForeignKey(Division, related_name="team's division")
	team_captain  = models.ForeignKey(Player, related_name="team_captain")
	other_players = models.ManyToManyField(Player, blank=True)

	def __str__(self):
		return self.name

'''
A division must have a name and a division rep and can contain 0 or more teams and 0 or more subs
Division -> Player  : 1 .. *
Division -> Team    : 0 .. *
Division -> Session : 1 .. *
'''
class Division(models.Model):
	name         = models.CharField(max_length=200)
	division_rep = models.ForeignKey(Player, related_name='division_representative')
	teams        = models.ManyToManyField(Team, blank=True, related_name="divisions_teams")

	def __str__(self):
		return self.name

'''
Sessions are generally named after a season (summer, fall, etc), they have an associated game and
division, as well as a start date and an end date.
Session -> Division : 1 .. 1
'''
class Session(models.Model):
	name       = models.CharField(max_length=200)
	game       = models.CharField(max_length=100)
	division   = models.ForeignKey(Division)
	start_date = models.DateTimeField('start date')
	end_date   = models.DateTimeField('end date')
	subs       = models.ManyToManyField(Sub, blank=True)
	#sublist    = models.OneToOneField('sublist.Sublist', blank=True)

	def __str__(self):
		return self.division.name + '_' + self.name + '_' + self.game

	def _get_sublist(self):
		return self.sublist
	sublist = property(_get_sublist)

<commit_msg>removed excess comments from main/models.py<commit_after>from django.db import models
from django.contrib.auth.models import User
from sublist.models import Sublist

'''
A player can exist on many teams and in many divisions both as a division rep and/or as a sub, and these
are not mutually exlusive
Player -> Team     : 0 .. *
Player -> Division : 0 .. *
Player -> Sub      :
'''
class Player(models.Model):
	user         = models.OneToOneField(
	    User,
            on_delete=models.CASCADE,
            primary_key=True,
	    )
	phone_number = models.IntegerField(blank=True, null=True)

	def __str__(self):
# chose not to use this because fName and lName are not required on users yet.
#		name = self.user.first_name + ' ' + self.user.last_name
		return self.user.username

	#function to return all of the sublists related to a player instance
	def related_sublists(self):
            return Sublist.objects.filter(session__subs__player=self)

'''
Subs are a "tuple-ish" construction tying a player and a date together to indicate what date they are
willing to sub in a Division.

Note: Eventually, I'd like players to be able to select a particular date that they could sub in a
Division, OR, put themselves down for an entire session, in which case, we will likely need a model for
session containing the dates of a particular session within a division. There are typically 3 sessions
in a division per year, but the sessions don't necessarily correspond across divisions.
'''
class Sub(models.Model):
	player = models.ForeignKey(Player)
	date   = models.DateTimeField('sub date')

	def __str__(self):
		availability = self.player.user.username + ' is available ' + str(self.date)
		return availability

	def _get_sessions(self):
		return self.session_set.all()
	sessions = property(_get_sessions)

'''
A team can contain many players but should only ever exist in one division
Team -> Player   : 1 .. *
Team -> Division : 1
'''
class Team(models.Model):
	name          = models.CharField(max_length=200)
#	division      = models.ForeignKey(Division, related_name="team's division")
	team_captain  = models.ForeignKey(Player, related_name="team_captain")
	other_players = models.ManyToManyField(Player, blank=True)

	def __str__(self):
		return self.name

'''
A division must have a name and a division rep and can contain 0 or more teams and 0 or more subs
Division -> Player  : 1 .. *
Division -> Team    : 0 .. *
Division -> Session : 1 .. *
'''
class Division(models.Model):
	name         = models.CharField(max_length=200)
	division_rep = models.ForeignKey(Player, related_name='division_representative')
	teams        = models.ManyToManyField(Team, blank=True, related_name="divisions_teams")

	def __str__(self):
		return self.name

'''
Sessions are generally named after a season (summer, fall, etc), they have an associated game and
division, as well as a start date and an end date.
Session -> Division : 1 .. 1
'''
class Session(models.Model):
	name       = models.CharField(max_length=200)
	game       = models.CharField(max_length=100)
	division   = models.ForeignKey(Division)
	start_date = models.DateTimeField('start date')
	end_date   = models.DateTimeField('end date')
	subs       = models.ManyToManyField(Sub, blank=True)
	#sublist    = models.OneToOneField('sublist.Sublist', blank=True)

	def __str__(self):
		return self.division.name + '_' + self.name + '_' + self.game

	def _get_sublist(self):
		return self.sublist
	sublist = property(_get_sublist)

<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import mock
from jsonschema import ValidationError

from st2common.content import utils as content_utils
from st2common.bootstrap.base import ResourceRegistrar
from st2common.persistence.pack import Pack
from st2common.persistence.pack import ConfigSchema

from st2tests.base import CleanDbTestCase
from st2tests import fixturesloader

__all__ = [
    'ResourceRegistrarTestCase'
]

PACK_PATH_1 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_1')
PACK_PATH_6 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_6')
PACK_PATH_7 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_7')
PACK_PATH_8 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_8')
PACK_PATH_9 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_9')
PACK_PATH_10 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_10')
PACK_PATH_11 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_11')
PACK_PATH_12 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_12')

class ResourceRegistrarTestCase(CleanDbTestCase):
    def test_register_packs(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        config_schema_dbs = ConfigSchema.get_all()

        self.assertEqual(len(pack_dbs), 0)
        self.assertEqual(len(config_schema_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Verify pack and schema have been registered
        pack_dbs = Pack.get_all()
        config_schema_dbs = ConfigSchema.get_all()

        self.assertEqual(len(pack_dbs), 1)
        self.assertEqual(len(config_schema_dbs), 1)

        self.assertEqual(pack_dbs[0].name, 'dummy_pack_1')
        self.assertTrue('api_key' in config_schema_dbs[0].attributes)
        self.assertTrue('api_secret' in config_schema_dbs[0].attributes)

    def test_register_pack_pack_ref(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()

        self.assertEqual(len(pack_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {
            'dummy_pack_1': PACK_PATH_1,
            'dummy_pack_6': PACK_PATH_6
        }
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Ref is provided
        pack_db = Pack.get_by_name('dummy_pack_6')
        self.assertEqual(pack_db.ref, 'dummy_pack_6_ref')

        # Ref is not provided, directory name should be used
        pack_db = Pack.get_by_name('dummy_pack_1')
        self.assertEqual(pack_db.ref, 'dummy_pack_1')

        # "ref" is not provided, but "name" is
        registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7)

        pack_db = Pack.get_by_name('dummy_pack_7_name')
        self.assertEqual(pack_db.ref, 'dummy_pack_7_name')

        # "ref" is not provided and "name" contains invalid characters
        expected_msg = 'contains invalid characters'
        self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_8)

    def test_register_pack_invalid_semver_version_friendly_error_message(self):
        registrar = ResourceRegistrar(use_pack_cache=False)

        expected_msg = ('Pack version "0.1.2.3.4" doesn\'t follow a valid semver format. Valid '
                        'versions and formats include: 0.1.0, 0.2.1, 1.1.0, etc.')
        self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_12)

        expected_msg = ('Pack version "0.2" doesn\'t follow a valid semver format. Valid '
                        'versions and formats include: 0.1.0, 0.2.1, 1.1.0, etc.')
        self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_11)

    @mock.patch('st2common.bootstrap.base.get_stackstorm_version')
    def test_register_pack_stackstorm_version_identifier_check(self, mock_get_stackstorm_version):
        # Version is satisfied
        mock_get_stackstorm_version.return_value = '2.0.0'

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_9': PACK_PATH_9}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        pack_db = Pack.get_by_name('dummy_pack_9_deps')
        self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0')

        # Pack requires a version which is not satisfied by current StackStorm version
        mock_get_stackstorm_version.return_value = '2.2.0'
        expected_msg = ('Pack "dummy_pack_9_deps" requires StackStorm ">=1.6.0, <2.2.0", but '
                        'current version is "2.2.0"')
        self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_9)

        mock_get_stackstorm_version.return_value = '2.3.0'
        expected_msg = ('Pack "dummy_pack_9_deps" requires StackStorm ">=1.6.0, <2.2.0", but '
                        'current version is "2.3.0"')
        self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_9)

        mock_get_stackstorm_version.return_value = '1.5.9'
        expected_msg = ('Pack "dummy_pack_9_deps" requires StackStorm ">=1.6.0, <2.2.0", but '
                        'current version is "1.5.9"')
        self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_9)

        mock_get_stackstorm_version.return_value = '1.5.0'
        expected_msg = ('Pack "dummy_pack_9_deps" requires StackStorm ">=1.6.0, <2.2.0", but '
                        'current version is "1.5.0"')
        self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_9)

    def test_register_pack_pack_stackstorm_version_and_future_parameters(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        self.assertEqual(len(pack_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_9': PACK_PATH_9}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Dependencies, stackstorm_version and future values
        pack_db = Pack.get_by_name('dummy_pack_9_deps')
        self.assertEqual(pack_db.dependencies, ['core=0.2.0'])
        self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0')
        self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}})

        # Note: We only store paramters which are defined in the schema, all other custom user
        # defined attributes are ignored
        self.assertTrue(not hasattr(pack_db, 'future'))
        self.assertTrue(not hasattr(pack_db, 'this'))

        # Wrong characters in the required st2 version
        expected_msg = "'wrongstackstormversion' does not match"
        self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_10)
<commit_msg>Remove version checks<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import mock
from jsonschema import ValidationError

from st2common.content import utils as content_utils
from st2common.bootstrap.base import ResourceRegistrar
from st2common.persistence.pack import Pack
from st2common.persistence.pack import ConfigSchema

from st2tests.base import CleanDbTestCase
from st2tests import fixturesloader

__all__ = [
    'ResourceRegistrarTestCase'
]

PACK_PATH_1 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_1')
PACK_PATH_6 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_6')
PACK_PATH_7 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_7')
PACK_PATH_8 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_8')
PACK_PATH_9 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_9')
PACK_PATH_10 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_10')
PACK_PATH_11 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_11')
PACK_PATH_12 = os.path.join(fixturesloader.get_fixtures_packs_base_path(), 'dummy_pack_12')

class ResourceRegistrarTestCase(CleanDbTestCase):
    def test_register_packs(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        config_schema_dbs = ConfigSchema.get_all()

        self.assertEqual(len(pack_dbs), 0)
        self.assertEqual(len(config_schema_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Verify pack and schema have been registered
        pack_dbs = Pack.get_all()
        config_schema_dbs = ConfigSchema.get_all()

        self.assertEqual(len(pack_dbs), 1)
        self.assertEqual(len(config_schema_dbs), 1)

        self.assertEqual(pack_dbs[0].name, 'dummy_pack_1')
        self.assertTrue('api_key' in config_schema_dbs[0].attributes)
        self.assertTrue('api_secret' in config_schema_dbs[0].attributes)

    def test_register_pack_pack_ref(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()

        self.assertEqual(len(pack_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {
            'dummy_pack_1': PACK_PATH_1,
            'dummy_pack_6': PACK_PATH_6
        }
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Ref is provided
        pack_db = Pack.get_by_name('dummy_pack_6')
        self.assertEqual(pack_db.ref, 'dummy_pack_6_ref')

        # Ref is not provided, directory name should be used
        pack_db = Pack.get_by_name('dummy_pack_1')
        self.assertEqual(pack_db.ref, 'dummy_pack_1')

        # "ref" is not provided, but "name" is
        registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7)

        pack_db = Pack.get_by_name('dummy_pack_7_name')
        self.assertEqual(pack_db.ref, 'dummy_pack_7_name')

        # "ref" is not provided and "name" contains invalid characters
        expected_msg = 'contains invalid characters'
        self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_8)

    def test_register_pack_invalid_semver_version_friendly_error_message(self):
        registrar = ResourceRegistrar(use_pack_cache=False)

        expected_msg = ('Pack version "0.1.2.3.4" doesn\'t follow a valid semver format. Valid '
                        'versions and formats include: 0.1.0, 0.2.1, 1.1.0, etc.')
        self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_12)

        expected_msg = ('Pack version "0.2" doesn\'t follow a valid semver format. Valid '
                        'versions and formats include: 0.1.0, 0.2.1, 1.1.0, etc.')
        self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_11)

    def test_register_pack_pack_stackstorm_version_and_future_parameters(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        self.assertEqual(len(pack_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_9': PACK_PATH_9}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Dependencies, stackstorm_version and future values
        pack_db = Pack.get_by_name('dummy_pack_9_deps')
        self.assertEqual(pack_db.dependencies, ['core=0.2.0'])
        self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0')
        self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}})

        # Note: We only store paramters which are defined in the schema, all other custom user
        # defined attributes are ignored
        self.assertTrue(not hasattr(pack_db, 'future'))
        self.assertTrue(not hasattr(pack_db, 'this'))

        # Wrong characters in the required st2 version
        expected_msg = "'wrongstackstormversion' does not match"
        self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_10)
<|endoftext|>


<commit_before>
# db uri string for mysql db. form: 'mysql+mysqldb://username:password@server_address/db_name'
SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://root@localhost/db'
TEST_SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://root@localhost/test'

WTF_CSRF_ENABLED = False

SECRET_KEY = 'generate a secret key and place here'

TWILIO_ACCOUNT_SID = "Your Twilio Account SID Number Here"
TWILIO_AUTH_TOKEN = "Your Twilio Auth Token Here"
TWILIO_NUMBER = "Your Twilio Number Here"
<commit_msg>updated default settings example<commit_after>
# db uri string for mysql db. form: 'mysql+mysqldb://username:password@server_address/db_name'
SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://steerclear:St33rCl3@r@localhost/db'
TEST_SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://steerclear:St33rCl3@r@localhost/test'

WTF_CSRF_ENABLED = False

SECRET_KEY = 'generate a secret key and place here'

TWILIO_ACCOUNT_SID = "Your Twilio Account SID Number Here"
TWILIO_AUTH_TOKEN = "Your Twilio Auth Token Here"
TWILIO_NUMBER = "Your Twilio Number Here"
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Ambari Agent

"""

__all__ = ["non_blocking_call", "checked_call", "call", "quote_bash_args", "as_user", "as_sudo"]

import time
import copy
import os
import select
import sys
import logging
import string
import subprocess
import threading
import traceback
from exceptions import Fail
from exceptions import ExecuteTimeoutException
from resource_management.core.logger import Logger
from ambari_commons.constants import AMBARI_SUDO_BINARY

# use quiet=True calls from this folder (logs get too messy duplicating the resources with its commands)
RMF_FOLDER = 'resource_management/'
EXPORT_PLACEHOLDER = "[RMF_EXPORT_PLACEHOLDER]"
ENV_PLACEHOLDER = "[RMF_ENV_PLACEHOLDER]"

PLACEHOLDERS_TO_STR = {
  EXPORT_PLACEHOLDER: "export {env_str} > /dev/null ; ",
  ENV_PLACEHOLDER: "{env_str}"
}

def log_function_call(function):
  def inner(command, **kwargs):
    caller_filename = sys._getframe(1).f_code.co_filename
    # quiet = can be False/True or None -- which means undefined yet
    quiet = kwargs['quiet'] if 'quiet' in kwargs else None
    is_internal_call = RMF_FOLDER in caller_filename

    if quiet == False or (quiet == None and not is_internal_call):
      command_alias = string_cmd_from_args_list(command) if isinstance(command, (list, tuple)) else command
      log_msg = Logger.get_function_repr("{0}['{1}']".format(function.__name__, command_alias), kwargs)
      Logger.info(log_msg)

    # logoutput=False - never log
    # logoutput=True - log in INFO level
    # logouput=None - log in DEBUG level
    # logouput=not-specified - log in DEBUG level, not counting internal calls
    kwargs['logoutput'] = ('logoutput' in kwargs and kwargs['logoutput'] and Logger.logger.isEnabledFor(logging.INFO)) or \
      ('logoutput' in kwargs and kwargs['logoutput']==None and Logger.logger.isEnabledFor(logging.DEBUG)) or \
      (not 'logoutput' in kwargs and not is_internal_call and Logger.logger.isEnabledFor(logging.DEBUG))

    result = function(command, **kwargs)

    if quiet == False or (quiet == None and not is_internal_call):
      log_msg = "{0} returned {1}".format(function.__name__, result)
      Logger.info(log_msg)

    return result

  return inner

@log_function_call
def checked_call(command, quiet=False, logoutput=None, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
         path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
  """
  Execute the shell command and throw an exception on failure.
  @throws Fail
  @return: return_code, output
  """
  return _call_wrapper(command, logoutput=logoutput, throw_on_failure=True, stdout=stdout, stderr=stderr,
                              cwd=cwd, env=env, preexec_fn=preexec_fn, user=user, wait_for_finish=wait_for_finish,
                              on_timeout=on_timeout, timeout=timeout, path=path, sudo=sudo, on_new_line=on_new_line,
                              tries=tries, try_sleep=try_sleep)

@log_function_call
def call(command, quiet=False, logoutput=None, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
         path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
  """
  Execute the shell command despite failures.
  @return: return_code, output
  """
  return _call_wrapper(command, logoutput=logoutput, throw_on_failure=False, stdout=stdout, stderr=stderr,
                              cwd=cwd, env=env, preexec_fn=preexec_fn, user=user, wait_for_finish=wait_for_finish,
                              on_timeout=on_timeout, timeout=timeout, path=path, sudo=sudo, on_new_line=on_new_line,
                              tries=tries, try_sleep=try_sleep)

@log_function_call
def non_blocking_call(command, quiet=False, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
         cwd=None, env=None, preexec_fn=None, user=None, timeout=None, path=None, sudo=False):
  """
  Execute the shell command and don't wait until it's completion

  @return: process object -- Popen instance
  (use proc.stdout.readline to read output in cycle, don't foget to proc.stdout.close(),
  to get return code use proc.wait() and after that proc.returncode)
  """
  return _call_wrapper(command, logoutput=False, throw_on_failure=True, stdout=stdout, stderr=stderr,
                              cwd=cwd, env=env, preexec_fn=preexec_fn, user=user, wait_for_finish=False,
                              on_timeout=None, timeout=timeout, path=path, sudo=sudo, on_new_line=None,
                              tries=1, try_sleep=0)

def _call_wrapper(command, **kwargs):
  tries = kwargs['tries']
  try_sleep = kwargs['try_sleep']
  timeout = kwargs['timeout']
  on_timeout = kwargs['on_timeout']
  throw_on_failure = kwargs['throw_on_failure']

  for i in range (0, tries):
    is_last_try = (i == tries-1)

    if not is_last_try:
      kwargs_copy = copy.copy(kwargs)
      kwargs_copy['throw_on_failure'] = True # we need to know when non-last try fails, to handle retries
    else:
      kwargs_copy = kwargs

    try:
      try:
        result = _call(command, **kwargs_copy)
        break
      except ExecuteTimeoutException as ex:
        if on_timeout:
          Logger.info("Executing '%s'. Reason: %s" % (on_timeout, str(ex)))
          result = checked_call(on_timeout)
        else:
          raise
    except Fail as ex:
      if is_last_try: # last try
        raise
      else:
        Logger.info("Retrying after %d seconds. Reason: %s" % (try_sleep, str(ex)))
        time.sleep(try_sleep)

  return result

def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
         path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
  """
  Execute shell command

  @param command: list/tuple of arguments (recommended as more safe - don't need to escape)
  or string of the command to execute
  @param logoutput: boolean, whether command output should be logged of not
  @param throw_on_failure: if true, when return code is not zero exception is thrown
  @param stdout,stderr:
    subprocess.PIPE - enable output to variable
    subprocess.STDOUT - redirect to stdout
    None - disable output to variable, and output to Python out straightly (even if logoutput is False)
    {int fd} - redirect to file with descriptor.
    {string filename} - redirects to a file with name.
  """
  command_alias = string_cmd_from_args_list(command) if isinstance(command, (list, tuple)) else command

  # Append current PATH to env['PATH']
  env = _add_current_path_to_env(env)
  # Append path to env['PATH']
  if path:
    path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path
    env['PATH'] = os.pathsep.join([env['PATH'], path])

  # prepare command cmd
  if sudo:
    command = as_sudo(command, env=env)
  elif user:
    command = as_user(command, user, env=env)

  # convert to string and escape
  if isinstance(command, (list, tuple)):
    command = string_cmd_from_args_list(command)

  # replace placeholder from as_sudo / as_user if present
  env_str = _get_environment_str(env)
  for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
    command = command.replace(placeholder, replacement.format(env_str=env_str))

  # --noprofile is used to preserve PATH set for ambari-agent
  subprocess_command = ["/bin/bash","--login","--noprofile","-c", command]

  files_to_close = []
  if isinstance(stdout, (basestring)):
    stdout = open(stdout, 'wb')
    files_to_close.append(stdout)
  if isinstance(stderr, (basestring)):
    stderr = open(stderr, 'wb')
    files_to_close.append(stderr)

  try:
    proc = subprocess.Popen(subprocess_command, stdout=stdout, stderr=stderr,
                            cwd=cwd, env=env, shell=False, close_fds=True,
                            preexec_fn=preexec_fn)

    if timeout:
      timeout_happened=False
      start = time.time()
      end = start+timeout

    if not wait_for_finish:
      return proc

    # in case logoutput==False, never log.
    logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG)
    read_set = []

    if stdout == subprocess.PIPE:
      read_set.append(proc.stdout)
    if stderr == subprocess.PIPE:
      read_set.append(proc.stderr)

    fd_to_string = {
      proc.stdout: "",
      proc.stderr: ""
    }
    all_output = ""

    while read_set:
      if timeout and time.time()> end:
        timeout_happened=True
        proc.kill()
        break
      ready, _, _ = select.select(read_set, [], [])
      for out_fd in read_set:
        if out_fd in ready:
          line = os.read(out_fd.fileno(), 1024)

          if not line:
            read_set = copy.copy(read_set)
            read_set.remove(out_fd)
            continue

          fd_to_string[out_fd] += line
          all_output += line

          if on_new_line:
            try:
              on_new_line(line, out_fd == proc.stderr)
            except Exception, err:
              err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(line, traceback.format_exc())
              raise Fail(err_msg)

          if logoutput:
            _print(line)

    # Wait for process to terminate
    while proc.poll() == None:
      if timeout and time.time()> end:
        timeout_happened=True
        proc.kill()
        break
      time.sleep(1)

  finally:
    for fp in files_to_close:
      fp.close()

  out = fd_to_string[proc.stdout].strip('\n')
  err = fd_to_string[proc.stderr].strip('\n')
  all_output = all_output.strip('\n')

  if timeout:
    if timeout_happened:
      err_msg = ("Execution of '%s' was killed due timeout after %d seconds") % (command, timeout)
      raise ExecuteTimeoutException(err_msg)

  code = proc.returncode

  if throw_on_failure and code:
    err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_alias, code, all_output))
    raise Fail(err_msg)

  # if separate stderr is enabled (by default it's redirected to out)
  if stderr == subprocess.PIPE:
    return code, out, err

  return code, out

def as_sudo(command, env=None, auto_escape=True):
  """
  command - list or tuple of arguments.
  env - when run as part of Execute resource, this SHOULD NOT be used.
  It automatically gets replaced later by call, checked_call. This should be used in not_if, only_if
  """
  if isinstance(command, (list, tuple)):
    command = string_cmd_from_args_list(command, auto_escape=auto_escape)
  else:
    # Since ambari user sudoer privileges may be restricted,
    # without having /bin/bash permission, and /bin/su permission.
    # Running interpreted shell commands in scope of 'sudo' is not possible.
    #
    # In that case while passing string,
    # any bash symbols eventually added to command like && || ; < > | << >> would cause problems.
    err_msg = Logger.filter_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple of arguments") % (command))
    raise Fail(err_msg)

  env = _get_environment_str(_add_current_path_to_env(env)) if env else ENV_PLACEHOLDER
  return "{0} {1} -H -E {2}".format(_get_sudo_binary(), env, command)

def as_user(command, user, env=None, auto_escape=True):
  if isinstance(command, (list, tuple)):
    command = string_cmd_from_args_list(command, auto_escape=auto_escape)

  export_env = "export {0} ; ".format(_get_environment_str(_add_current_path_to_env(env))) if env else EXPORT_PLACEHOLDER
  return "{0} su {1} -l -s /bin/bash -c {2}".format(_get_sudo_binary(), user, quote_bash_args(export_env + command))

def quote_bash_args(command):
  if not command:
    return "''"

  if not isinstance(command, basestring):
    raise Fail("Command should be a list of strings, found '{0}' in command list elements".format(str(command)))

  valid = set(string.ascii_letters + string.digits + '@%_-+=:,./')
  for char in command:
    if char not in valid:
      return "'" + command.replace("'", "'\"'\"'") + "'"
  return command

def _add_current_path_to_env(env):
  result = {} if not env else env

  if not 'PATH' in result:
    result['PATH'] = os.environ['PATH']

  # don't append current env if already there
  if not set(os.environ['PATH'].split(os.pathsep)).issubset(result['PATH'].split(os.pathsep)):
    result['PATH'] = os.pathsep.join([os.environ['PATH'], result['PATH']])

  return result

def _get_sudo_binary():
  return AMBARI_SUDO_BINARY

def _get_environment_str(env):
  return reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, '')

def string_cmd_from_args_list(command, auto_escape=True):
  escape_func = lambda x:quote_bash_args(x) if auto_escape else lambda x:x
  return ' '.join(escape_func(x) for x in command)

def _print(line):
  sys.stdout.write(line)
  sys.stdout.flush()<commit_msg>AMBARI-11956. HBase Master stop is hanging when enabling security (Ivan Kozlov via smohanty)<commit_after>#!/usr/bin/env python
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Ambari Agent

"""

__all__ = ["non_blocking_call", "checked_call", "call", "quote_bash_args", "as_user", "as_sudo"]

import time
import copy
import os
import select
import sys
import logging
import string
import subprocess
import threading
import traceback
from exceptions import Fail
from exceptions import ExecuteTimeoutException
from resource_management.core.logger import Logger
from ambari_commons.constants import AMBARI_SUDO_BINARY

# use quiet=True calls from this folder (logs get too messy duplicating the resources with its commands)
RMF_FOLDER = 'resource_management/'
EXPORT_PLACEHOLDER = "[RMF_EXPORT_PLACEHOLDER]"
ENV_PLACEHOLDER = "[RMF_ENV_PLACEHOLDER]"

PLACEHOLDERS_TO_STR = {
  EXPORT_PLACEHOLDER: "export {env_str} > /dev/null ; ",
  ENV_PLACEHOLDER: "{env_str}"
}

def log_function_call(function):
  def inner(command, **kwargs):
    caller_filename = sys._getframe(1).f_code.co_filename
    # quiet = can be False/True or None -- which means undefined yet
    quiet = kwargs['quiet'] if 'quiet' in kwargs else None
    is_internal_call = RMF_FOLDER in caller_filename

    if quiet == False or (quiet == None and not is_internal_call):
      command_alias = string_cmd_from_args_list(command) if isinstance(command, (list, tuple)) else command
      log_msg = Logger.get_function_repr("{0}['{1}']".format(function.__name__, command_alias), kwargs)
      Logger.info(log_msg)

    # logoutput=False - never log
    # logoutput=True - log in INFO level
    # logouput=None - log in DEBUG level
    # logouput=not-specified - log in DEBUG level, not counting internal calls
    kwargs['logoutput'] = ('logoutput' in kwargs and kwargs['logoutput'] and Logger.logger.isEnabledFor(logging.INFO)) or \
      ('logoutput' in kwargs and kwargs['logoutput']==None and Logger.logger.isEnabledFor(logging.DEBUG)) or \
      (not 'logoutput' in kwargs and not is_internal_call and Logger.logger.isEnabledFor(logging.DEBUG))

    result = function(command, **kwargs)

    if quiet == False or (quiet == None and not is_internal_call):
      log_msg = "{0} returned {1}".format(function.__name__, result)
      Logger.info(log_msg)

    return result

  return inner

@log_function_call
def checked_call(command, quiet=False, logoutput=None, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
         path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
  """
  Execute the shell command and throw an exception on failure.
  @throws Fail
  @return: return_code, output
  """
  return _call_wrapper(command, logoutput=logoutput, throw_on_failure=True, stdout=stdout, stderr=stderr,
                              cwd=cwd, env=env, preexec_fn=preexec_fn, user=user, wait_for_finish=wait_for_finish,
                              on_timeout=on_timeout, timeout=timeout, path=path, sudo=sudo, on_new_line=on_new_line,
                              tries=tries, try_sleep=try_sleep)

@log_function_call
def call(command, quiet=False, logoutput=None, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
         path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
  """
  Execute the shell command despite failures.
  @return: return_code, output
  """
  return _call_wrapper(command, logoutput=logoutput, throw_on_failure=False, stdout=stdout, stderr=stderr,
                              cwd=cwd, env=env, preexec_fn=preexec_fn, user=user, wait_for_finish=wait_for_finish,
                              on_timeout=on_timeout, timeout=timeout, path=path, sudo=sudo, on_new_line=on_new_line,
                              tries=tries, try_sleep=try_sleep)

@log_function_call
def non_blocking_call(command, quiet=False, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
         cwd=None, env=None, preexec_fn=None, user=None, timeout=None, path=None, sudo=False):
  """
  Execute the shell command and don't wait until it's completion

  @return: process object -- Popen instance
  (use proc.stdout.readline to read output in cycle, don't foget to proc.stdout.close(),
  to get return code use proc.wait() and after that proc.returncode)
  """
  return _call_wrapper(command, logoutput=False, throw_on_failure=True, stdout=stdout, stderr=stderr,
                              cwd=cwd, env=env, preexec_fn=preexec_fn, user=user, wait_for_finish=False,
                              on_timeout=None, timeout=timeout, path=path, sudo=sudo, on_new_line=None,
                              tries=1, try_sleep=0)

def _call_wrapper(command, **kwargs):
  tries = kwargs['tries']
  try_sleep = kwargs['try_sleep']
  timeout = kwargs['timeout']
  on_timeout = kwargs['on_timeout']
  throw_on_failure = kwargs['throw_on_failure']

  for i in range (0, tries):
    is_last_try = (i == tries-1)

    if not is_last_try:
      kwargs_copy = copy.copy(kwargs)
      kwargs_copy['throw_on_failure'] = True # we need to know when non-last try fails, to handle retries
    else:
      kwargs_copy = kwargs

    try:
      try:
        result = _call(command, **kwargs_copy)
        break
      except ExecuteTimeoutException as ex:
        if on_timeout:
          Logger.info("Executing '%s'. Reason: %s" % (on_timeout, str(ex)))
          result = checked_call(on_timeout)
        else:
          raise
    except Fail as ex:
      if is_last_try: # last try
        raise
      else:
        Logger.info("Retrying after %d seconds. Reason: %s" % (try_sleep, str(ex)))
        time.sleep(try_sleep)

  return result

def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
         path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
  """
  Execute shell command

  @param command: list/tuple of arguments (recommended as more safe - don't need to escape)
  or string of the command to execute
  @param logoutput: boolean, whether command output should be logged of not
  @param throw_on_failure: if true, when return code is not zero exception is thrown
  @param stdout,stderr:
    subprocess.PIPE - enable output to variable
    subprocess.STDOUT - redirect to stdout
    None - disable output to variable, and output to Python out straightly (even if logoutput is False)
    {int fd} - redirect to file with descriptor.
    {string filename} - redirects to a file with name.
  """
  command_alias = string_cmd_from_args_list(command) if isinstance(command, (list, tuple)) else command

  # Append current PATH to env['PATH']
  env = _add_current_path_to_env(env)
  # Append path to env['PATH']
  if path:
    path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path
    env['PATH'] = os.pathsep.join([env['PATH'], path])

  # prepare command cmd
  if sudo:
    command = as_sudo(command, env=env)
  elif user:
    command = as_user(command, user, env=env)

  # convert to string and escape
  if isinstance(command, (list, tuple)):
    command = string_cmd_from_args_list(command)

  # replace placeholder from as_sudo / as_user if present
  env_str = _get_environment_str(env)
  for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
    command = command.replace(placeholder, replacement.format(env_str=env_str))

  # --noprofile is used to preserve PATH set for ambari-agent
  subprocess_command = ["/bin/bash","--login","--noprofile","-c", command]

  files_to_close = []
  if isinstance(stdout, (basestring)):
    stdout = open(stdout, 'wb')
    files_to_close.append(stdout)
  if isinstance(stderr, (basestring)):
    stderr = open(stderr, 'wb')
    files_to_close.append(stderr)

  try:
    proc = subprocess.Popen(subprocess_command, stdout=stdout, stderr=stderr,
                            cwd=cwd, env=env, shell=False, close_fds=True,
                            preexec_fn=preexec_fn)

    if timeout:
      timeout_event = threading.Event()
      t = threading.Timer( timeout, _on_timeout, [proc, timeout_event] )
      t.start()

    if not wait_for_finish:
      return proc

    # in case logoutput==False, never log.
    logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG)
    read_set = []

    if stdout == subprocess.PIPE:
      read_set.append(proc.stdout)
    if stderr == subprocess.PIPE:
      read_set.append(proc.stderr)

    fd_to_string = {
      proc.stdout: "",
      proc.stderr: ""
    }
    all_output = ""

    while read_set:

      is_proccess_running = (proc.poll() == None)
      ready, _, _ = select.select(read_set, [], [], 1)

      if not is_proccess_running and not ready:
        break

      for out_fd in read_set:
        if out_fd in ready:
          line = os.read(out_fd.fileno(), 1024)

          if not line:
            read_set = copy.copy(read_set)
            read_set.remove(out_fd)
            out_fd.close()
            continue

          fd_to_string[out_fd] += line
          all_output += line

          if on_new_line:
            try:
              on_new_line(line, out_fd == proc.stderr)
            except Exception, err:
              err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(line, traceback.format_exc())
              raise Fail(err_msg)

          if logoutput:
            _print(line)

    # Wait for process to terminate
    proc.wait()

  finally:
    for fp in files_to_close:
      fp.close()

  out = fd_to_string[proc.stdout].strip('\n')
  err = fd_to_string[proc.stderr].strip('\n')
  all_output = all_output.strip('\n')

  if timeout:
    if not timeout_event.is_set():
      t.cancel()
    # timeout occurred
    else:
      err_msg = ("Execution of '%s' was killed due timeout after %d seconds") % (command, timeout)
      raise ExecuteTimeoutException(err_msg)

  code = proc.returncode

  if throw_on_failure and code:
    err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_alias, code, all_output))
    raise Fail(err_msg)

  # if separate stderr is enabled (by default it's redirected to out)
  if stderr == subprocess.PIPE:
    return code, out, err

  return code, out

def as_sudo(command, env=None, auto_escape=True):
  """
  command - list or tuple of arguments.
  env - when run as part of Execute resource, this SHOULD NOT be used.
  It automatically gets replaced later by call, checked_call. This should be used in not_if, only_if
  """
  if isinstance(command, (list, tuple)):
    command = string_cmd_from_args_list(command, auto_escape=auto_escape)
  else:
    # Since ambari user sudoer privileges may be restricted,
    # without having /bin/bash permission, and /bin/su permission.
    # Running interpreted shell commands in scope of 'sudo' is not possible.
    #
    # In that case while passing string,
    # any bash symbols eventually added to command like && || ; < > | << >> would cause problems.
    err_msg = Logger.filter_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple of arguments") % (command))
    raise Fail(err_msg)

  env = _get_environment_str(_add_current_path_to_env(env)) if env else ENV_PLACEHOLDER
  return "{0} {1} -H -E {2}".format(_get_sudo_binary(), env, command)

def as_user(command, user, env=None, auto_escape=True):
  if isinstance(command, (list, tuple)):
    command = string_cmd_from_args_list(command, auto_escape=auto_escape)

  export_env = "export {0} ; ".format(_get_environment_str(_add_current_path_to_env(env))) if env else EXPORT_PLACEHOLDER
  return "{0} su {1} -l -s /bin/bash -c {2}".format(_get_sudo_binary(), user, quote_bash_args(export_env + command))

def quote_bash_args(command):
  if not command:
    return "''"

  if not isinstance(command, basestring):
    raise Fail("Command should be a list of strings, found '{0}' in command list elements".format(str(command)))

  valid = set(string.ascii_letters + string.digits + '@%_-+=:,./')
  for char in command:
    if char not in valid:
      return "'" + command.replace("'", "'\"'\"'") + "'"
  return command

def _add_current_path_to_env(env):
  result = {} if not env else env

  if not 'PATH' in result:
    result['PATH'] = os.environ['PATH']

  # don't append current env if already there
  if not set(os.environ['PATH'].split(os.pathsep)).issubset(result['PATH'].split(os.pathsep)):
    result['PATH'] = os.pathsep.join([os.environ['PATH'], result['PATH']])

  return result

def _get_sudo_binary():
  return AMBARI_SUDO_BINARY

def _get_environment_str(env):
  return reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, '')

def string_cmd_from_args_list(command, auto_escape=True):
  escape_func = lambda x:quote_bash_args(x) if auto_escape else lambda x:x
  return ' '.join(escape_func(x) for x in command)

def _print(line):
  sys.stdout.write(line)
  sys.stdout.flush()

def _on_timeout(proc, timeout_event):
  timeout_event.set()
  if proc.poll() == None:
    try:
      proc.terminate()
      proc.wait()
    # catch race condition if proc already dead
    except OSError:
      pass

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Oct 25 14:49:32 2016

@author: Dave Ho
"""

#HOST&&PORT: http://127.0.0.1:5000/

import sqlite3
import flask
from flask import Flask
from flask import request
import ast

app = Flask(__name__)

DATABASE = 'database.db'

@app.route("/", methods=["GET", "POST"])
def hello():
    #if request.method == "GET":
    print "someone said get"
    return "JJ!"
    '''
    if request.method == "POST":
        content = request.get_json(silent=True)
        print content
        print "someone posted something"
        return ""
    '''

@app.route("/json", methods=['GET', 'POST'])
def json():
    dickeys = request.form.keys()
    dic = ast.literal_eval(dickeys[0])
    conn = sqlite3.connect('database.db')
    c = conn.cursor()
    try:
        c.execute("INSERT INTO users VALUES('"+dic['phone']+"', '"+dic['phone']+"')")
    except sqlite3.IntegrityError:
        pass
    #c.execute("INSERT INTO users VALUES('"+dic['phone']+"', '"+dic['phone']+"')")
    # Needs try and except block
    c.execute('SELECT COUNT(pid) FROM path')
    count = c.fetchone()[0]
    c.execute("INSERT INTO path VALUES('"+str(count)+"', '"+dic['phone']+"', '"+dic['title']+"', '"+dic['zipCodeList'][0]+"')")
    for i in range(0, len(dic['lat'])):
        c.execute("INSERT INTO points VALUES('"+str(count)+"', '"+str(dic['lat'][i])+"', '"+str(dic['lng'][i])+"', '"+str(i)+"')")
    for i in range(0, len(dic['markerMap'].keys())):
        key = sorted(dic['markerMap'].keys())[i]
        c.execute("INSERT INTO markers VALUES('"+str(count)+"', '"+str(dic['markerMap'][key]['lat'])+"', '"+str(dic['markerMap'][key]['lng'])+"', '"+dic['markerMap'][key]['description']+"', '"+dic['markerMap'][key]['image']+"')")
    conn.commit()
    conn.close()
    return request.json

@app.route("/getTitle", methods=["GET", "POST"])
def getTitle():
    """
    Example JSON Call: http://192.168.1.65/getTitle?zip=95135
    Android User Given: Zip Code of Current User Location
    What Server Needs to return: Given zip code of current user location, give back set of paths within that location using json
    use paths table
    """
    if request.method == "GET":
        conn = sqlite3.connect('database.db')
        c = conn.cursor()
        zipcode = str(request.args.get('zip'))
        # need to return titles back to android user from database
        resultString = ""
        for row in c.execute("SELECT * FROM path WHERE zip="+zipcode):
            resultString += str(row[1])+"_"+str(row[2])+"\n"
        print resultString
        print "someone got some detail"
        return resultString

@app.route("/getDetail", methods=["GET", "POST"])
def getDetail():
    """
    Example JSON Call: http://192.168.1.65/getDetail?phonenumber=14083346432&id=hi
    Android User Given: Phone + title of path selected
    What Server Needs to return: given phone and title of path return the dictionary json of that path
    user users table
    """
    if request.method == "GET":
        conn = sqlite3.connect('database.db')
        c = conn.cursor()
        # need to return full details back to android user from database
        phone = str(request.args.get('phonenumber'))
        titleid = str(request.args.get('id'))
        print phone
        print titleid
        c.execute('SELECT pid FROM paths WHERE user='+phone+' AND title='+titleid)
        path_id = c.fetchone()[0]
        points_list = []
        for row in c.execute('SELECT lat, lng FROM points WHERE pid='+path_id+"ORDER BY 'order'"):
            points_list.append(row)
        markers = c.execute('SELECT * FROM markers WHERE pid='+path_id)
        print "someone got some title"
        return "You have gotten some title"

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=80)
    #app.run(host='0.0.0.0',port="80")<commit_msg>finish debugging server/getDetail 3/3<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Oct 25 14:49:32 2016

@author: Dave Ho
"""

#HOST&&PORT: http://127.0.0.1:5000/

import sqlite3
import flask
from flask import Flask
from flask import request
import ast

app = Flask(__name__)

DATABASE = 'database.db'

@app.route("/", methods=["GET", "POST"])
def hello():
    #if request.method == "GET":
    print "someone said get"
    return "JJ!"
    '''
    if request.method == "POST":
        content = request.get_json(silent=True)
        print content
        print "someone posted something"
        return ""
    '''

@app.route("/json", methods=['GET', 'POST'])
def json():
    dickeys = request.form.keys()
    dic = ast.literal_eval(dickeys[0])
    conn = sqlite3.connect('database.db')
    c = conn.cursor()
    try:
        c.execute("INSERT INTO users VALUES('"+dic['phone']+"', '"+dic['phone']+"')")
    except sqlite3.IntegrityError:
        pass
    #c.execute("INSERT INTO users VALUES('"+dic['phone']+"', '"+dic['phone']+"')")
    # Needs try and except block
    c.execute('SELECT COUNT(pid) FROM path')
    count = c.fetchone()[0]
    c.execute("INSERT INTO path VALUES('"+str(count)+"', '"+dic['phone']+"', '"+dic['title']+"', '"+dic['zipCodeList'][0]+"')")
    for i in range(0, len(dic['lat'])):
        c.execute("INSERT INTO points VALUES('"+str(count)+"', '"+str(dic['lat'][i])+"', '"+str(dic['lng'][i])+"', '"+str(i)+"')")
    for i in range(0, len(dic['markerMap'].keys())):
        key = sorted(dic['markerMap'].keys())[i]
        c.execute("INSERT INTO markers VALUES('"+str(count)+"', '"+str(dic['markerMap'][key]['lat'])+"', '"+str(dic['markerMap'][key]['lng'])+"', '"+dic['markerMap'][key]['description']+"', '"+dic['markerMap'][key]['image']+"')")
    conn.commit()
    conn.close()
    return request.json

@app.route("/getTitle", methods=["GET", "POST"])
def getTitle():
    """
    Example JSON Call: http://192.168.1.65/getTitle?zip=95135
    Android User Given: Zip Code of Current User Location
    What Server Needs to return: Given zip code of current user location, give back set of paths within that location using json
    use paths table
    """
    if request.method == "GET":
        conn = sqlite3.connect('database.db')
        c = conn.cursor()
        zipcode = str(request.args.get('zip'))
        # need to return titles back to android user from database
        resultString = ""
        for row in c.execute("SELECT * FROM path WHERE zip="+zipcode):
            resultString += str(row[1])+"_"+str(row[2])+"\n"
        print resultString
        print "someone got some detail"
        return resultString

@app.route("/getDetail", methods=["GET", "POST"])
def getDetail():
    """
    Example JSON Call: http://192.168.1.65/getDetail?phonenumber=14083346432&id=hi
    Android User Given: Phone + title of path selected
    What Server Needs to return: given phone and title of path return the dictionary json of that path
    user users table
    """
    if request.method == "GET":
        conn = sqlite3.connect('database.db')
        c = conn.cursor()
        # need to return full details back to android user from database
        phone = str(request.args.get('phonenumber'))
        titleid = str(request.args.get('id'))
        print phone
        print titleid
        c.execute('SELECT pid FROM paths WHERE user='+phone+' AND title='+titleid)
        path_id = c.fetchone()[0]
        points_list = []
        for row in c.execute('SELECT lat, lng FROM points WHERE pid='+path_id+"ORDER BY 'order'"):
            points_list.append(row)
        marker_list = []
        for row in c.execute('SELECT lat, lng, description, image FROM markers WHERE pid='+path_id+"ORDER BY 'order'"):
            marker_list.append(row)
        print "someone got some title"
        return "You have gotten some title"

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=80)
    #app.run(host='0.0.0.0',port="80")<|endoftext|>


<commit_before>from sqlalchemy import Column
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import create_engine
from sqlalchemy import Date
from sqlalchemy.dialects.mysql import DOUBLE
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker

Base = declarative_base()
class Multimedia(Base):
    __tablename__ = 'multimedia'

    media_id     = Column(Integer, primary_key=True, \
                          autoincrement=True, nullable=False)
    title        = Column(String(100), nullable=False)
    creator      = Column(String(100), nullable=False)
    description  = Column(String(100), nullable=False)
    media_type   = Column(String(100), nullable=False)
    number_of_comments = Column(Integer, nullable=False, default=0)

<commit_msg>deleted import statements<commit_after>from sqlalchemy import Column
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()
class Multimedia(Base):
    __tablename__ = 'multimedia'

    media_id     = Column(Integer, primary_key=True,
                          autoincrement=True, nullable=False)
    title        = Column(String(100), nullable=False)
    creator      = Column(String(100), nullable=False)
    description  = Column(String(100), nullable=False)
    media_type   = Column(String(100), nullable=False)
    number_of_comments = Column(Integer, nullable=False, default=0)

<|endoftext|>